From f376347f24f24efc157d80de63381dd22d060630 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 5 Jun 2018 17:58:28 +0200 Subject: Find and mark more Git disk access locations, part 2 --- lib/gitlab/git/gitlab_projects.rb | 37 +----------------- lib/gitlab/setup_helper.rb | 4 +- lib/gitlab/shell.rb | 59 ++++++++++++++++++++++++++--- lib/gitlab/task_helpers.rb | 4 +- lib/system_check/orphans/namespace_check.rb | 16 ++++---- lib/tasks/gitlab/check.rake | 12 +++--- lib/tasks/gitlab/info.rake | 6 ++- 7 files changed, 82 insertions(+), 56 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index 00c943fdb25..8475645971e 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -53,24 +53,11 @@ module Gitlab # Import project via git clone --bare # URL must be publicly cloneable def import_project(source, timeout) - Gitlab::GitalyClient.migrate(:import_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_import_repository(source) - else - git_import_repository(source, timeout) - end - end + git_import_repository(source, timeout) end def fork_repository(new_shard_name, new_repository_relative_path) - Gitlab::GitalyClient.migrate(:fork_repository, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_fork_repository(new_shard_name, new_repository_relative_path) - else - git_fork_repository(new_shard_name, new_repository_relative_path) - end - end + git_fork_repository(new_shard_name, new_repository_relative_path) end def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) @@ -241,16 +228,6 @@ module Gitlab true end - def gitaly_import_repository(source) - raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) - - Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source) - true - rescue GRPC::BadStatus => e - @output << e.message - false - end - def git_fork_repository(new_shard_name, new_repository_relative_path) from_path = repository_absolute_path new_shard_path = Gitlab.config.repositories.storages.fetch(new_shard_name).legacy_disk_path @@ -270,16 +247,6 @@ module Gitlab run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) end - - def gitaly_fork_repository(new_shard_name, new_repository_relative_path) - target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil) - raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) - - Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) - rescue GRPC::BadStatus => e - logger.error "fork-repository failed: #{e.message}" - false - end end end end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index e5c02dd8ecc..4a87f43597e 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -24,7 +24,9 @@ module Gitlab address = val['gitaly_address'] end - storages << { name: key, path: val.legacy_disk_path } + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + storages << { name: key, path: val.legacy_disk_path } + end end if Rails.env.test? diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 4a691d640b3..4b8aae4f5a2 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -106,10 +106,17 @@ module Gitlab raise Error.new("don't use disk paths with import_repository: #{url.inspect}") end - # The timeout ensures the subprocess won't hang forever - cmd = gitlab_projects(storage, "#{name}.git") - success = cmd.import_project(url, git_timeout) + relative_path = "#{name}.git" + cmd = gitaly_migrate(:import_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + if is_enabled + GitalyGitlabProjects.new(storage, relative_path) + else + # The timeout ensures the subprocess won't hang forever + gitlab_projects(storage, relative_path) + end + end + success = cmd.import_project(url, git_timeout) raise Error, cmd.output unless success success @@ -165,8 +172,16 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817 def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) - gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git") - .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git") + forked_from_relative_path = "#{forked_from_disk_path}.git" + fork_args = [forked_to_storage, "#{forked_to_disk_path}.git"] + + gitaly_migrate(:fork_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + if is_enabled + GitalyGitlabProjects.new(forked_from_storage, forked_from_relative_path).fork_repository(*fork_args) + else + gitlab_projects(forked_from_storage, forked_from_relative_path).fork_repository(*fork_args) + end + end end # Removes a repository from file system, using rm_diretory which is an alias @@ -452,5 +467,39 @@ module Gitlab # need to do the same here... raise Error, e end + + class GitalyGitlabProjects + attr_reader :shard_name, :repository_relative_path, :output + + def initialize(shard_name, repository_relative_path) + @shard_name = shard_name + @repository_relative_path = repository_relative_path + @output = '' + end + + def import_project(source, _timeout) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source) + true + rescue GRPC::BadStatus => e + @output = e.message + false + end + + def fork_repository(new_shard_name, new_repository_relative_path) + target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) + rescue GRPC::BadStatus => e + logger.error "fork-repository failed: #{e.message}" + false + end + + def logger + Rails.logger + end + end end end diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 723e655c150..922418966e9 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -140,7 +140,9 @@ module Gitlab end def repository_storage_paths_args - Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } + end end def user_home diff --git a/lib/system_check/orphans/namespace_check.rb b/lib/system_check/orphans/namespace_check.rb index b5f443abe06..09b57c7b408 100644 --- a/lib/system_check/orphans/namespace_check.rb +++ b/lib/system_check/orphans/namespace_check.rb @@ -4,13 +4,15 @@ module SystemCheck set_name 'Orphaned namespaces:' def multi_check - Gitlab.config.repositories.storages.each do |storage_name, repository_storage| - $stdout.puts - $stdout.puts "* Storage: #{storage_name} (#{repository_storage.legacy_disk_path})".color(:yellow) - toplevel_namespace_dirs = disk_namespaces(repository_storage.legacy_disk_path) - - orphans = (toplevel_namespace_dirs - existing_namespaces) - print_orphans(orphans, storage_name) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + $stdout.puts + $stdout.puts "* Storage: #{storage_name} (#{repository_storage.legacy_disk_path})".color(:yellow) + toplevel_namespace_dirs = disk_namespaces(repository_storage.legacy_disk_path) + + orphans = (toplevel_namespace_dirs - existing_namespaces) + print_orphans(orphans, storage_name) + end end clear_namespaces! # releases memory when check finishes diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index c04dae7446f..a8acafa9cd9 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -44,11 +44,13 @@ namespace :gitlab do start_checking "GitLab Shell" check_gitlab_shell - check_repo_base_exists - check_repo_base_is_not_symlink - check_repo_base_user_and_group - check_repo_base_permissions - check_repos_hooks_directory_is_link + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + check_repo_base_exists + check_repo_base_is_not_symlink + check_repo_base_user_and_group + check_repo_base_permissions + check_repos_hooks_directory_is_link + end check_gitlab_shell_self_test finished_checking "GitLab Shell" diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 289aa5d9060..6de739e9515 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -67,8 +67,10 @@ namespace :gitlab do puts "GitLab Shell".color(:yellow) puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" puts "Repository storage paths:" - Gitlab.config.repositories.storages.each do |name, repository_storage| - puts "- #{name}: \t#{repository_storage.legacy_disk_path}" + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.each do |name, repository_storage| + puts "- #{name}: \t#{repository_storage.legacy_disk_path}" + end end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" -- cgit v1.2.1