diff options
| -rw-r--r-- | GITALY_SERVER_VERSION | 2 | ||||
| -rw-r--r-- | Gemfile | 2 | ||||
| -rw-r--r-- | Gemfile.lock | 4 | ||||
| -rw-r--r-- | Gemfile.rails5.lock | 4 | ||||
| -rw-r--r-- | changelogs/unreleased/use-backup-custom-hooks-gitaly.yml | 5 | ||||
| -rw-r--r-- | lib/backup/repository.rb | 130 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client/repository_service.rb | 42 |
7 files changed, 119 insertions, 70 deletions
diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index c64d9d48a48..a9a7f3fec01 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -0.105.1 +0.107.0 @@ -419,7 +419,7 @@ group :ed25519 do end # Gitaly GRPC client -gem 'gitaly-proto', '~> 0.101.0', require: 'gitaly' +gem 'gitaly-proto', '~> 0.102.0', require: 'gitaly' gem 'grpc', '~> 1.11.0' # Locked until https://github.com/google/protobuf/issues/4210 is closed diff --git a/Gemfile.lock b/Gemfile.lock index 883e580b86b..8d508cc0f39 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -283,7 +283,7 @@ GEM gettext_i18n_rails (>= 0.7.1) po_to_json (>= 1.0.0) rails (>= 3.2.0) - gitaly-proto (0.101.0) + gitaly-proto (0.102.0) google-protobuf (~> 3.1) grpc (~> 1.10) github-linguist (5.3.3) @@ -1039,7 +1039,7 @@ DEPENDENCIES gettext (~> 3.2.2) gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails_js (~> 1.3) - gitaly-proto (~> 0.101.0) + gitaly-proto (~> 0.102.0) github-linguist (~> 5.3.3) gitlab-flowdock-git-hook (~> 1.0.1) gitlab-gollum-lib (~> 4.2) diff --git a/Gemfile.rails5.lock b/Gemfile.rails5.lock index 223717f1818..fd560290d6c 100644 --- a/Gemfile.rails5.lock +++ b/Gemfile.rails5.lock @@ -286,7 +286,7 @@ GEM gettext_i18n_rails (>= 0.7.1) po_to_json (>= 1.0.0) rails (>= 3.2.0) - gitaly-proto (0.101.0) + gitaly-proto (0.102.0) google-protobuf (~> 3.1) grpc (~> 1.10) github-linguist (5.3.3) @@ -1049,7 +1049,7 @@ DEPENDENCIES gettext (~> 3.2.2) gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails_js (~> 1.3) - gitaly-proto (~> 0.101.0) + gitaly-proto (~> 0.102.0) github-linguist (~> 5.3.3) gitlab-flowdock-git-hook (~> 1.0.1) gitlab-gollum-lib (~> 4.2) diff --git a/changelogs/unreleased/use-backup-custom-hooks-gitaly.yml b/changelogs/unreleased/use-backup-custom-hooks-gitaly.yml new file mode 100644 index 00000000000..4b9766332c3 --- /dev/null +++ b/changelogs/unreleased/use-backup-custom-hooks-gitaly.yml @@ -0,0 +1,5 @@ +--- +title: migrate backup rake task to gitaly +merge_request: +author: +type: added diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 0119c5d6851..221ba52b490 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -4,7 +4,6 @@ require_relative 'helper' module Backup class Repository include Backup::Helper - # rubocop:disable Metrics/AbcSize attr_reader :progress @@ -18,61 +17,26 @@ module Backup Project.find_each(batch_size: 1000) do |project| progress.print " * #{display_repo_path(project)} ... " - path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(project) - end - path_to_project_bundle = path_to_bundle(project) - - # Create namespace dir or hashed path if missing if project.hashed_storage?(:repository) FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path))) else FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace end - if empty_repo?(project) - progress.puts "[SKIPPED]".color(:cyan) + if !empty_repo?(project) + backup_project(project) + progress.puts "[DONE]".color(:green) else - in_path(path_to_project_repo) do |dir| - FileUtils.mkdir_p(path_to_tars(project)) - cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) - output, status = Gitlab::Popen.popen(cmd) - - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end - - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) - output, status = Gitlab::Popen.popen(cmd) - - if status.zero? - progress.puts "[DONE]".color(:green) - else - progress_warn(project, cmd.join(' '), output) - end + progress.puts "[SKIPPED]".color(:cyan) end wiki = ProjectWiki.new(project) - path_to_wiki_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(wiki) - end - path_to_wiki_bundle = path_to_bundle(wiki) - if File.exist?(path_to_wiki_repo) - progress.print " * #{display_repo_path(wiki)} ... " - - if empty_repo?(wiki) - progress.puts " [SKIPPED]".color(:cyan) - else - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all) - output, status = Gitlab::Popen.popen(cmd) - if status.zero? - progress.puts " [DONE]".color(:green) - else - progress_warn(wiki, cmd.join(' '), output) - end - end + if !empty_repo?(wiki) + backup_project(wiki) + progress.puts "[DONE] Wiki".color(:green) + else + progress.puts "[SKIPPED] Wiki".color(:cyan) end end end @@ -83,6 +47,38 @@ module Backup end end + def backup_project(project) + gitaly_migrate(:repository_backup) do |is_enabled| + if is_enabled + backup_project_gitaly(project) + else + backup_project_local(project) + end + end + + backup_custom_hooks(project) + rescue => e + progress_warn(project, e, 'Failed to backup repo') + end + + def backup_project_gitaly(project) + path_to_project_bundle = path_to_bundle(project) + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .create_bundle(path_to_project_bundle) + end + + def backup_project_local(project) + path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + path_to_repo(project) + end + + path_to_project_bundle = path_to_bundle(project) + + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) + output, status = Gitlab::Popen.popen(cmd) + progress_warn(project, cmd.join(' '), output) unless status.zero? + end + def delete_all_repositories(name, repository_storage) gitaly_migrate(:delete_all_repositories) do |is_enabled| if is_enabled @@ -97,8 +93,6 @@ module Backup path = repository_storage.legacy_disk_path return unless File.exist?(path) - # Move all files in the existing repos directory except . and .. to - # repositories.old.<timestamp> directory bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) FileUtils.mkdir_p(bk_repos_path, mode: 0700) files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] @@ -129,13 +123,47 @@ module Backup .restore_custom_hooks(custom_hooks_path) end + def local_backup_custom_hooks(project) + in_path(path_to_tars(project)) do |dir| + path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + path_to_repo(project) + end + break unless File.exist?(File.join(path_to_project_repo, dir)) + + FileUtils.mkdir_p(path_to_tars(project)) + cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir}) + output, status = Gitlab::Popen.popen(cmd) + + unless status.zero? + progress_warn(project, cmd.join(' '), output) + end + end + end + + def gitaly_backup_custom_hooks(project) + FileUtils.mkdir_p(path_to_tars(project)) + custom_hooks_path = path_to_tars(project, 'custom_hooks') + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .backup_custom_hooks(custom_hooks_path) + end + + def backup_custom_hooks(project) + gitaly_migrate(:backup_custom_hooks) do |is_enabled| + if is_enabled + gitaly_backup_custom_hooks(project) + else + local_backup_custom_hooks(project) + end + end + end + def restore_custom_hooks(project) in_path(path_to_tars(project)) do |dir| gitaly_migrate(:restore_custom_hooks) do |is_enabled| if is_enabled - local_restore_custom_hooks(project, dir) - else gitaly_restore_custom_hooks(project, dir) + else + local_restore_custom_hooks(project, dir) end end end @@ -186,7 +214,6 @@ module Backup end end end - # rubocop:enable Metrics/AbcSize protected @@ -224,9 +251,7 @@ module Backup def prepare FileUtils.rm_rf(backup_repos_path) - # Ensure the parent dir of backup_repos_path exists FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create backup_repos_path before us FileUtils.mkdir(backup_repos_path, mode: 0700) end @@ -242,7 +267,6 @@ module Backup end def empty_repo?(project_or_wiki) - # Protect against stale caches project_or_wiki.repository.expire_emptiness_caches project_or_wiki.repository.empty? end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 4340f779e53..ca986434221 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -196,20 +196,21 @@ module Gitlab end def create_bundle(save_path) - request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo) - response = GitalyClient.call( - @storage, - :repository_service, + gitaly_fetch_stream_to_file( + save_path, :create_bundle, - request, - timeout: GitalyClient.default_timeout + Gitaly::CreateBundleRequest, + GitalyClient.default_timeout ) + end - File.open(save_path, 'wb') do |f| - response.each do |message| - f.write(message.data) - end - end + def backup_custom_hooks(save_path) + gitaly_fetch_stream_to_file( + save_path, + :backup_custom_hooks, + Gitaly::BackupCustomHooksRequest, + GitalyClient.default_timeout + ) end def create_from_bundle(bundle_path) @@ -309,6 +310,25 @@ module Gitlab private + def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout) + request = request_class.new(repository: @gitaly_repo) + response = GitalyClient.call( + @storage, + :repository_service, + rpc_name, + request, + timeout: timeout + ) + + File.open(save_path, 'wb') do |f| + response.each do |message| + f.write(message.data) + end + end + # If the file is empty means that we recieved an empty stream, we delete the file + FileUtils.rm(save_path) if File.zero?(save_path) + end + def gitaly_repo_stream_request(file_path, rpc_name, request_class, timeout) request = request_class.new(repository: @gitaly_repo) enum = Enumerator.new do |y| |