shell.rb 14 KB
Newer Older
1 2
# frozen_string_literal: true

Zeger-Jan van de Weg's avatar
Zeger-Jan van de Weg committed
3
# Gitaly note: SSH key operations are not part of Gitaly so will never be migrated.
Jacob Vosmaer's avatar
Jacob Vosmaer committed
4

5 6
require 'securerandom'

7
module Gitlab
8
  class Shell
9 10
    GITLAB_SHELL_ENV_VARS = %w(GIT_TERMINAL_PROMPT).freeze

11
    Error = Class.new(StandardError)
12

Gabriel Mazetto's avatar
Gabriel Mazetto committed
13
    KeyAdder = Struct.new(:io) do
14
      def add_key(id, key)
15 16 17 18 19 20
        key = Gitlab::Shell.strip_key(key)
        # Newline and tab are part of the 'protocol' used to transmit id+key to the other end
        if key.include?("\t") || key.include?("\n")
          raise Error.new("Invalid key: #{key.inspect}")
        end

21
        io.puts("#{id}\t#{key}")
22 23 24
      end
    end

25
    class << self
26 27 28 29 30 31 32 33 34 35 36 37
      def secret_token
        @secret_token ||= begin
          File.read(Gitlab.config.gitlab_shell.secret_file).chomp
        end
      end

      def ensure_secret_token!
        return if File.exist?(File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret'))

        generate_and_link_secret_token
      end

38
      def version_required
39 40
        @version_required ||= File.read(Rails.root
                                        .join('GITLAB_SHELL_VERSION')).strip
41
      end
42 43

      def strip_key(key)
44
        key.split(/[ ]+/)[0, 2].join(' ')
45
      end
46 47 48 49 50 51 52 53 54 55

      private

      # Create (if necessary) and link the secret token file
      def generate_and_link_secret_token
        secret_file = Gitlab.config.gitlab_shell.secret_file
        shell_path = Gitlab.config.gitlab_shell.path

        unless File.size?(secret_file)
          # Generate a new token of 16 random hexadecimal characters and store it in secret_file.
56 57
          @secret_token = SecureRandom.hex(16)
          File.write(secret_file, @secret_token)
58 59 60 61 62 63 64
        end

        link_path = File.join(shell_path, '.gitlab_shell_secret')
        if File.exist?(shell_path) && !File.exist?(link_path)
          FileUtils.symlink(secret_file, link_path)
        end
      end
65 66
    end

67
    # Convenience methods for initializing a new repository with a Project model.
68 69 70 71 72 73 74 75
    def create_project_repository(project)
      create_repository(project.repository_storage, project.disk_path, project.full_path)
    end

    def create_wiki_repository(project)
      create_repository(project.repository_storage, project.wiki.disk_path, project.wiki.full_path)
    end

76
    # Init new repository
77
    #
78
    # storage - the shard key
79
    # disk_path - project disk path
80
    # gl_project_path - project name
81 82
    #
    # Ex.
83
    #   create_repository("default", "path/to/gitlab-ci", "gitlab/gitlab-ci")
84
    #
85
    def create_repository(storage, disk_path, gl_project_path)
86
      relative_path = disk_path.dup
Jacob Vosmaer's avatar
Jacob Vosmaer committed
87 88
      relative_path << '.git' unless relative_path.end_with?('.git')

89 90 91
      # During creation of a repository, gl_repository may not be known
      # because that depends on a yet-to-be assigned project ID in the
      # database (e.g. project-1234), so for now it is blank.
92
      repository = Gitlab::Git::Repository.new(storage, relative_path, '', gl_project_path)
Jacob Vosmaer's avatar
Jacob Vosmaer committed
93 94 95
      wrapped_gitaly_errors { repository.gitaly_repository_client.create_repository }

      true
96
    rescue => err # Once the Rugged codes gets removes this can be improved
97
      Rails.logger.error("Failed to add repository #{storage}/#{disk_path}: #{err}")
98
      false
99 100
    end

101 102 103 104 105 106 107 108
    def import_wiki_repository(project, wiki_formatter)
      import_repository(project.repository_storage, wiki_formatter.disk_path, wiki_formatter.import_url, project.wiki.full_path)
    end

    def import_project_repository(project)
      import_repository(project.repository_storage, project.disk_path, project.import_url, project.full_path)
    end

109 110
    # Import repository
    #
111
    # storage - project's storage name
112 113
    # name - project disk path
    # url - URL to import from
114 115
    #
    # Ex.
116
    #   import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git")
117
    #
118
    def import_repository(storage, name, url, gl_project_path)
Jacob Vosmaer's avatar
rubocop  
Jacob Vosmaer committed
119
      if url.start_with?('.', '/')
Jacob Vosmaer's avatar
Jacob Vosmaer committed
120 121 122
        raise Error.new("don't use disk paths with import_repository: #{url.inspect}")
      end

123
      relative_path = "#{name}.git"
124
      cmd = GitalyGitlabProjects.new(storage, relative_path, gl_project_path)
125

126
      success = cmd.import_project(url, git_timeout)
127 128 129
      raise Error, cmd.output unless success

      success
130 131
    end

132 133 134 135
    # Move repository reroutes to mv_directory which is an alias for
    # mv_namespace. Given the underlying implementation is a move action,
    # indescriminate of what the folders might be.
    #
136
    # storage - project's storage path
137 138
    # path - project disk path
    # new_path - new project disk path
139 140
    #
    # Ex.
141 142
    #   mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
    def mv_repository(storage, path, new_path)
143 144 145
      return false if path.empty? || new_path.empty?

      !!mv_directory(storage, "#{path}.git", "#{new_path}.git")
146 147
    end

148
    # Fork repository to new path
Stan Hu's avatar
Stan Hu committed
149 150 151 152 153 154 155
    # source_project - forked-from Project
    # target_project - forked-to Project
    def fork_repository(source_project, target_project)
      forked_from_relative_path = "#{source_project.disk_path}.git"
      fork_args = [target_project.repository_storage, "#{target_project.disk_path}.git", target_project.full_path]

      GitalyGitlabProjects.new(source_project.repository_storage, forked_from_relative_path, source_project.full_path).fork_repository(*fork_args)
156 157
    end

158 159 160
    # Removes a repository from file system, using rm_diretory which is an alias
    # for rm_namespace. Given the underlying implementation removes the name
    # passed as second argument on the passed storage.
161
    #
162
    # storage - project's storage path
163
    # name - project disk path
164 165
    #
    # Ex.
166 167
    #   remove_repository("/path/to/storage", "gitlab/gitlab-ci")
    def remove_repository(storage, name)
168 169 170 171 172 173
      return false if name.empty?

      !!rm_directory(storage, "#{name}.git")
    rescue ArgumentError => e
      Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git")
      false
Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
174 175
    end

176
    # Add new key to gitlab-shell
177
    #
178
    # Ex.
179
    #   add_key("key-42", "sha-rsa ...")
180
    #
181
    def add_key(key_id, key_content)
182 183
      return unless self.authorized_keys_enabled?

184 185
      gitlab_shell_fast_execute([gitlab_shell_keys_path,
                                 'add-key', key_id, self.class.strip_key(key_content)])
186 187
    end

188 189 190 191 192
    # Batch-add keys to authorized_keys
    #
    # Ex.
    #   batch_add_keys { |adder| adder.add_key("key-42", "sha-rsa ...") }
    def batch_add_keys(&block)
193 194
      return unless self.authorized_keys_enabled?

195
      IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys batch-add-keys), 'w') do |io|
196
        yield(KeyAdder.new(io))
197 198 199
      end
    end

200
    # Remove ssh key from gitlab shell
201 202
    #
    # Ex.
203
    #   remove_key("key-342", "sha-rsa ...")
204
    #
205
    def remove_key(key_id, key_content = nil)
206 207
      return unless self.authorized_keys_enabled?

208 209 210
      args = [gitlab_shell_keys_path, 'rm-key', key_id]
      args << key_content if key_content
      gitlab_shell_fast_execute(args)
211 212
    end

213 214 215
    # Remove all ssh keys from gitlab shell
    #
    # Ex.
Johannes Schleifenbaum's avatar
Johannes Schleifenbaum committed
216
    #   remove_all_keys
217 218
    #
    def remove_all_keys
219 220
      return unless self.authorized_keys_enabled?

221
      gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear'])
222 223
    end

224 225 226 227 228
    # Remove ssh keys from gitlab shell that are not in the DB
    #
    # Ex.
    #   remove_keys_not_found_in_db
    #
229
    # rubocop: disable CodeReuse/ActiveRecord
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
    def remove_keys_not_found_in_db
      return unless self.authorized_keys_enabled?

      Rails.logger.info("Removing keys not found in DB")

      batch_read_key_ids do |ids_in_file|
        ids_in_file.uniq!
        keys_in_db = Key.where(id: ids_in_file)

        next unless ids_in_file.size > keys_in_db.count # optimization

        ids_to_remove = ids_in_file - keys_in_db.pluck(:id)
        ids_to_remove.each do |id|
          Rails.logger.info("Removing key-#{id} not found in DB")
          remove_key("key-#{id}")
        end
      end
    end
248
    # rubocop: enable CodeReuse/ActiveRecord
249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276

    # Iterate over all ssh key IDs from gitlab shell, in batches
    #
    # Ex.
    #   batch_read_key_ids { |batch| keys = Key.where(id: batch) }
    #
    def batch_read_key_ids(batch_size: 100, &block)
      return unless self.authorized_keys_enabled?

      list_key_ids do |key_id_stream|
        key_id_stream.lazy.each_slice(batch_size) do |lines|
          key_ids = lines.map { |l| l.chomp.to_i }
          yield(key_ids)
        end
      end
    end

    # Stream all ssh key IDs from gitlab shell, separated by newlines
    #
    # Ex.
    #   list_key_ids
    #
    def list_key_ids(&block)
      return unless self.authorized_keys_enabled?

      IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys list-key-ids), &block)
    end

277 278 279
    # Add empty directory for storing repositories
    #
    # Ex.
280
    #   add_namespace("default", "gitlab")
281
    #
282
    def add_namespace(storage, name)
283 284 285 286
      # https://gitlab.com/gitlab-org/gitlab-ce/issues/58012
      Gitlab::GitalyClient.allow_n_plus_1_calls do
        Gitlab::GitalyClient::NamespaceService.new(storage).add(name)
      end
287 288
    rescue GRPC::InvalidArgument => e
      raise ArgumentError, e.message
289 290 291 292 293 294
    end

    # Remove directory from repositories storage
    # Every repository inside this directory will be removed too
    #
    # Ex.
295
    #   rm_namespace("default", "gitlab")
296
    #
297
    def rm_namespace(storage, name)
298
      Gitlab::GitalyClient::NamespaceService.new(storage).remove(name)
299 300
    rescue GRPC::InvalidArgument => e
      raise ArgumentError, e.message
301
    end
302
    alias_method :rm_directory, :rm_namespace
303 304 305 306

    # Move namespace directory inside repositories storage
    #
    # Ex.
307
    #   mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
308
    #
309
    def mv_namespace(storage, old_name, new_name)
310
      Gitlab::GitalyClient::NamespaceService.new(storage).rename(old_name, new_name)
311 312 313
    rescue GRPC::InvalidArgument => e
      Gitlab::Sentry.track_acceptable_exception(e, extra: { old_name: old_name, new_name: new_name, storage: storage })

314
      false
315
    end
316
    alias_method :mv_directory, :mv_namespace # Note: ShellWorker uses this alias
317

318
    def url_to_repo(path)
319
      Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git"
320
    end
321

322 323
    # Return GitLab shell version
    def version
324
      gitlab_shell_version_file = "#{gitlab_shell_path}/VERSION"
325 326

      if File.readable?(gitlab_shell_version_file)
327
        File.read(gitlab_shell_version_file).chomp
328 329 330
      end
    end

331 332 333
    # Check if such directory exists in repositories.
    #
    # Usage:
334 335
    #   exists?(storage, 'gitlab')
    #   exists?(storage, 'gitlab/cookies.git')
336
    #
337
    # rubocop: disable CodeReuse/ActiveRecord
338
    def exists?(storage, dir_name)
339
      Gitlab::GitalyClient::NamespaceService.new(storage).exists?(dir_name)
340
    end
341
    # rubocop: enable CodeReuse/ActiveRecord
342

343 344 345 346
    def hooks_path
      File.join(gitlab_shell_path, 'hooks')
    end

347 348
    protected

349
    def gitlab_shell_path
350 351 352
      File.expand_path(Gitlab.config.gitlab_shell.path)
    end

353 354 355 356
    def gitlab_shell_user_home
      File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}")
    end

357
    def full_path(storage, dir_name)
358 359
      raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?

360
      File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, dir_name)
361 362
    end

363 364 365 366 367 368 369
    def gitlab_shell_projects_path
      File.join(gitlab_shell_path, 'bin', 'gitlab-projects')
    end

    def gitlab_shell_keys_path
      File.join(gitlab_shell_path, 'bin', 'gitlab-keys')
    end
370

371 372 373 374 375 376 377 378
    def authorized_keys_enabled?
      # Return true if nil to ensure the authorized_keys methods work while
      # fixing the authorized_keys file during migration.
      return true if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled.nil?

      Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled
    end

379 380 381 382 383 384 385 386 387 388 389
    private

    def gitlab_shell_fast_execute(cmd)
      output, status = gitlab_shell_fast_execute_helper(cmd)

      return true if status.zero?

      Rails.logger.error("gitlab-shell failed with error #{status}: #{output}")
      false
    end

390 391
    def gitlab_shell_fast_execute_raise_error(cmd, vars = {})
      output, status = gitlab_shell_fast_execute_helper(cmd, vars)
392 393

      raise Error, output unless status.zero?
394

395 396 397
      true
    end

398 399
    def gitlab_shell_fast_execute_helper(cmd, vars = {})
      vars.merge!(ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS))
400 401 402 403 404

      # Don't pass along the entire parent environment to prevent gitlab-shell
      # from wasting I/O by searching through GEM_PATH
      Bundler.with_original_env { Popen.popen(cmd, nil, vars) }
    end
405

406 407 408 409
    def git_timeout
      Gitlab.config.gitlab_shell.git_timeout
    end

Jacob Vosmaer's avatar
Jacob Vosmaer committed
410 411
    def wrapped_gitaly_errors
      yield
412 413 414 415 416
    rescue GRPC::NotFound, GRPC::BadStatus => e
      # Old Popen code returns [Error, output] to the caller, so we
      # need to do the same here...
      raise Error, e
    end
417 418

    class GitalyGitlabProjects
419
      attr_reader :shard_name, :repository_relative_path, :output, :gl_project_path
420

421
      def initialize(shard_name, repository_relative_path, gl_project_path)
422 423 424
        @shard_name = shard_name
        @repository_relative_path = repository_relative_path
        @output = ''
425
        @gl_project_path = gl_project_path
426 427 428
      end

      def import_project(source, _timeout)
429
        raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil, gl_project_path)
430 431 432 433 434 435 436 437

        Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source)
        true
      rescue GRPC::BadStatus => e
        @output = e.message
        false
      end

438 439
      def fork_repository(new_shard_name, new_repository_relative_path, new_project_name)
        target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil, new_project_name)
440
        raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil, gl_project_path)
441 442 443 444 445 446 447 448 449 450 451

        Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository)
      rescue GRPC::BadStatus => e
        logger.error "fork-repository failed: #{e.message}"
        false
      end

      def logger
        Rails.logger
      end
    end
452 453
  end
end