diff options
Diffstat (limited to 'lib/gitlab/shell.rb')
| -rw-r--r-- | lib/gitlab/shell.rb | 152 |
1 files changed, 95 insertions, 57 deletions
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 9cdd3d22f18..f4a41dc3eda 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -71,7 +71,6 @@ module Gitlab # Ex. # add_repository("/path/to/storage", "gitlab/gitlab-ci") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) relative_path = name.dup relative_path << '.git' unless relative_path.end_with?('.git') @@ -100,8 +99,12 @@ module Gitlab # Ex. # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874 def import_repository(storage, name, url) + if url.start_with?('.', '/') + raise Error.new("don't use disk paths with import_repository: #{url.inspect}") + end + # The timeout ensures the subprocess won't hang forever cmd = gitlab_projects(storage, "#{name}.git") success = cmd.import_project(url, git_timeout) @@ -122,11 +125,10 @@ module Gitlab # Ex. # fetch_remote(my_repo, "upstream") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false) gitaly_migrate(:fetch_remote) do |is_enabled| if is_enabled - repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) + repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout) else storage_path = Gitlab.config.repositories.storages[repository.storage]["path"] local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) @@ -134,7 +136,10 @@ module Gitlab end end - # Move repository + # Move repository reroutes to mv_directory which is an alias for + # mv_namespace. Given the underlying implementation is a move action, + # indescriminate of what the folders might be. + # # storage - project's storage path # path - project disk path # new_path - new project disk path @@ -142,9 +147,11 @@ module Gitlab # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def mv_repository(storage, path, new_path) - gitlab_projects(storage, "#{path}.git").mv_project("#{new_path}.git") + return false if path.empty? || new_path.empty? + + !!mv_directory(storage, "#{path}.git", "#{new_path}.git") end # Fork repository to new path @@ -156,13 +163,15 @@ module Gitlab # Ex. # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci") # - # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817 def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git") .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git") end - # Remove repository from file system + # Removes a repository from file system, using rm_diretory which is an alias + # for rm_namespace. Given the underlying implementation removes the name + # passed as second argument on the passed storage. # # storage - project's storage path # name - project disk path @@ -170,9 +179,14 @@ module Gitlab # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def remove_repository(storage, name) - gitlab_projects(storage, "#{name}.git").rm_project + return false if name.empty? + + !!rm_directory(storage, "#{name}.git") + rescue ArgumentError => e + Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git") + false end # Add new key to gitlab-shell @@ -181,6 +195,8 @@ module Gitlab # add_key("key-42", "sha-rsa ...") # def add_key(key_id, key_content) + return unless self.authorized_keys_enabled? + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'add-key', key_id, self.class.strip_key(key_content)]) end @@ -190,6 +206,8 @@ module Gitlab # Ex. # batch_add_keys { |adder| adder.add_key("key-42", "sha-rsa ...") } def batch_add_keys(&block) + return unless self.authorized_keys_enabled? + IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys batch-add-keys), 'w') do |io| yield(KeyAdder.new(io)) end @@ -200,10 +218,11 @@ module Gitlab # Ex. # remove_key("key-342", "sha-rsa ...") # - def remove_key(key_id, key_content) + def remove_key(key_id, key_content = nil) + return unless self.authorized_keys_enabled? + args = [gitlab_shell_keys_path, 'rm-key', key_id] args << key_content if key_content - gitlab_shell_fast_execute(args) end @@ -213,15 +232,67 @@ module Gitlab # remove_all_keys # def remove_all_keys + return unless self.authorized_keys_enabled? + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear']) end + # Remove ssh keys from gitlab shell that are not in the DB + # + # Ex. + # remove_keys_not_found_in_db + # + def remove_keys_not_found_in_db + return unless self.authorized_keys_enabled? + + Rails.logger.info("Removing keys not found in DB") + + batch_read_key_ids do |ids_in_file| + ids_in_file.uniq! + keys_in_db = Key.where(id: ids_in_file) + + next unless ids_in_file.size > keys_in_db.count # optimization + + ids_to_remove = ids_in_file - keys_in_db.pluck(:id) + ids_to_remove.each do |id| + Rails.logger.info("Removing key-#{id} not found in DB") + remove_key("key-#{id}") + end + end + end + + # Iterate over all ssh key IDs from gitlab shell, in batches + # + # Ex. + # batch_read_key_ids { |batch| keys = Key.where(id: batch) } + # + def batch_read_key_ids(batch_size: 100, &block) + return unless self.authorized_keys_enabled? + + list_key_ids do |key_id_stream| + key_id_stream.lazy.each_slice(batch_size) do |lines| + key_ids = lines.map { |l| l.chomp.to_i } + yield(key_ids) + end + end + end + + # Stream all ssh key IDs from gitlab shell, separated by newlines + # + # Ex. + # list_key_ids + # + def list_key_ids(&block) + return unless self.authorized_keys_enabled? + + IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys list-key-ids), &block) + end + # Add empty directory for storing repositories # # Ex. # add_namespace("/path/to/storage", "gitlab") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) Gitlab::GitalyClient.migrate(:add_namespace) do |enabled| if enabled @@ -243,7 +314,6 @@ module Gitlab # Ex. # rm_namespace("/path/to/storage", "gitlab") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled| if enabled @@ -255,13 +325,13 @@ module Gitlab rescue GRPC::InvalidArgument => e raise ArgumentError, e.message end + alias_method :rm_directory, :rm_namespace # Move namespace directory inside repositories storage # # Ex. # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled| if enabled @@ -275,6 +345,7 @@ module Gitlab rescue GRPC::InvalidArgument false end + alias_method :mv_directory, :mv_namespace def url_to_repo(path) Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git" @@ -306,47 +377,6 @@ module Gitlab end end - # Push branch to remote repository - # - # storage - project's storage path - # project_name - project's disk path - # remote_name - remote name - # branch_names - remote branch names to push - # forced - should we use --force flag - # - # Ex. - # push_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test' 'upstream', ['feature']) - # - def push_remote_branches(storage, project_name, remote_name, branch_names, forced: true) - cmd = gitlab_projects(storage, "#{project_name}.git") - - success = cmd.push_branches(remote_name, git_timeout, forced, branch_names) - - raise Error, cmd.output unless success - - success - end - - # Delete branch from remote repository - # - # storage - project's storage path - # project_name - project's disk path - # remote_name - remote name - # branch_names - remote branch names - # - # Ex. - # delete_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test', 'upstream', ['feature']) - # - def delete_remote_branches(storage, project_name, remote_name, branch_names) - cmd = gitlab_projects(storage, "#{project_name}.git") - - success = cmd.delete_remote_branches(remote_name, branch_names) - - raise Error, cmd.output unless success - - success - end - protected def gitlab_shell_path @@ -375,6 +405,14 @@ module Gitlab File.join(gitlab_shell_path, 'bin', 'gitlab-keys') end + def authorized_keys_enabled? + # Return true if nil to ensure the authorized_keys methods work while + # fixing the authorized_keys file during migration. + return true if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled.nil? + + Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled + end + private def gitlab_projects(shard_path, disk_path) |