From 3b5e9f0608fac06b6c5f39a036efa3c1d82cc6f5 Mon Sep 17 00:00:00 2001 From: Ahmad Hassan Date: Sun, 29 Apr 2018 14:01:00 +0300 Subject: Use gitaly repository_service.CreateFromBundle to restore --- lib/backup/repository.rb | 55 ++---------------------------------------------- 1 file changed, 2 insertions(+), 53 deletions(-) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 65e06fd78c0..42d7fe570ec 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -91,65 +91,14 @@ module Backup prepare_directories Project.find_each(batch_size: 1000) do |project| progress.print " * #{display_repo_path(project)} ... " - path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) - project.ensure_storage_path_exists - - cmd = if File.exist?(path_to_project_bundle) - %W(#{Gitlab.config.git.bin_path} clone --bare --mirror #{path_to_project_bundle} #{path_to_project_repo}) - else - %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo}) - end - - output, status = Gitlab::Popen.popen(cmd) - if status.zero? - progress.puts "[DONE]".color(:green) - else - progress_warn(project, cmd.join(' '), output) - end - - in_path(path_to_tars(project)) do |dir| - cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) - - output, status = Gitlab::Popen.popen(cmd) - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end + project.repository.create_from_bundle path_to_project_bundle unless project.repository_exists? wiki = ProjectWiki.new(project) - path_to_wiki_repo = path_to_repo(wiki) path_to_wiki_bundle = path_to_bundle(wiki) - if File.exist?(path_to_wiki_bundle) - progress.print " * #{display_repo_path(wiki)} ... " - - # If a wiki bundle exists, first remove the empty repo - # that was initialized with ProjectWiki.new() and then - # try to restore with 'git clone --bare'. - FileUtils.rm_rf(path_to_wiki_repo) - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_wiki_bundle} #{path_to_wiki_repo}) - - output, status = Gitlab::Popen.popen(cmd) - if status.zero? - progress.puts " [DONE]".color(:green) - else - progress_warn(project, cmd.join(' '), output) - end - end - end - - progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) - cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args - - output, status = Gitlab::Popen.popen(cmd) - if status.zero? - progress.puts " [DONE]".color(:green) - else - puts " [FAILED]".color(:red) - puts "failed: #{cmd}" - puts output + project.repository.create_from_bundle(path_to_wiki_bundle) if File.exists?(path_to_wiki_bundle) end end # rubocop:enable Metrics/AbcSize -- cgit v1.2.1 From fb54eb76ee79121f2aeb480944e6d0552100743b Mon Sep 17 00:00:00 2001 From: Ahmad Hassan Date: Fri, 4 May 2018 16:02:01 +0200 Subject: Remove useless print line --- lib/backup/repository.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 42d7fe570ec..d068d1f66c4 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -90,7 +90,6 @@ module Backup def restore prepare_directories Project.find_each(batch_size: 1000) do |project| - progress.print " * #{display_repo_path(project)} ... " path_to_project_bundle = path_to_bundle(project) project.repository.create_from_bundle path_to_project_bundle unless project.repository_exists? -- cgit v1.2.1 From 845726212584fd9413e1fa70fb3896342f824434 Mon Sep 17 00:00:00 2001 From: Ahmad Hassan Date: Sun, 6 May 2018 21:32:25 +0200 Subject: Better repo restore progress logging --- lib/backup/repository.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index d068d1f66c4..4fc1dded344 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -93,11 +93,15 @@ module Backup path_to_project_bundle = path_to_bundle(project) project.repository.create_from_bundle path_to_project_bundle unless project.repository_exists? + progress.puts "[DONE] restoring #{project.name} repository".color(:green) wiki = ProjectWiki.new(project) path_to_wiki_bundle = path_to_bundle(wiki) - project.repository.create_from_bundle(path_to_wiki_bundle) if File.exists?(path_to_wiki_bundle) + if File.exists?(path_to_wiki_bundle) + project.repository.create_from_bundle(path_to_wiki_bundle) + progress.puts "[DONE] restoring #{project.name} wiki".color(:green) + end end end # rubocop:enable Metrics/AbcSize -- cgit v1.2.1 From 0734da16f631cc9f842bd5c0de08cc2b306738e5 Mon Sep 17 00:00:00 2001 From: Ahmad Hassan Date: Sun, 6 May 2018 21:42:39 +0200 Subject: Wrap create_from_bundle in a begin rescue block --- lib/backup/repository.rb | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 4fc1dded344..67e317c35bc 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -92,15 +92,29 @@ module Backup Project.find_each(batch_size: 1000) do |project| path_to_project_bundle = path_to_bundle(project) - project.repository.create_from_bundle path_to_project_bundle unless project.repository_exists? - progress.puts "[DONE] restoring #{project.name} repository".color(:green) + if File.exist?(path_to_project_bundle) + begin + project.repository.create_from_bundle path_to_project_bundle unless project.repository_exists? + progress.puts "[DONE] restoring #{project.name} repository".color(:green) + rescue StandardError => e + progress.puts "[Failed] restoring #{project.name} repository".color(:red) + progress.puts "Error: #{e}".color(:red) + end + else + progress.puts "[Failed] bundle file #{path_to_project_bundle} does not exist" + end wiki = ProjectWiki.new(project) path_to_wiki_bundle = path_to_bundle(wiki) - if File.exists?(path_to_wiki_bundle) - project.repository.create_from_bundle(path_to_wiki_bundle) - progress.puts "[DONE] restoring #{project.name} wiki".color(:green) + if File.exist?(path_to_wiki_bundle) + begin + project.repository.create_from_bundle(path_to_wiki_bundle) + progress.puts "[DONE] restoring #{project.name} wiki".color(:green) + rescue StandardError => e + progress.puts "[Failed] restoring #{project.name} wiki".color(:red) + progress.puts "Error #{e}".color(:red) + end end end end -- cgit v1.2.1 From 2bcc324f268942d28c427cd3208e43df03af957d Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Fri, 11 May 2018 11:33:29 -0700 Subject: Add back some of the non-gitaly restore functionality behind gates --- lib/backup/repository.rb | 86 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 20 deletions(-) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 67e317c35bc..74b9e0205a4 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -68,40 +68,70 @@ module Backup def prepare_directories Gitlab.config.repositories.storages.each do |name, repository_storage| - path = repository_storage.legacy_disk_path - next unless File.exist?(path) - - # Move all files in the existing repos directory except . and .. to - # repositories.old. directory - bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) - FileUtils.mkdir_p(bk_repos_path, mode: 0700) - files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] - - begin - FileUtils.mv(files, bk_repos_path) - rescue Errno::EACCES - access_denied_error(path) - rescue Errno::EBUSY - resource_busy_error(path) + gitaly_migrate(:remove_repositories) do |is_enabled| + # TODO: Need to find a way to do this for gitaly + unless is_enabled + path = repository_storage.legacy_disk_path + next unless File.exist?(path) + + # Move all files in the existing repos directory except . and .. to + # repositories.old. directory + bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) + FileUtils.mkdir_p(bk_repos_path, mode: 0700) + files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] + + begin + FileUtils.mv(files, bk_repos_path) + rescue Errno::EACCES + access_denied_error(path) + rescue Errno::EBUSY + resource_busy_error(path) + end + end end end end def restore prepare_directories + gitlab_shell = Gitlab::Shell.new Project.find_each(batch_size: 1000) do |project| path_to_project_bundle = path_to_bundle(project) + path_to_project_repo = path_to_repo(project) + project.ensure_storage_path_exists + restore_repo_status = nil if File.exist?(path_to_project_bundle) begin - project.repository.create_from_bundle path_to_project_bundle unless project.repository_exists? - progress.puts "[DONE] restoring #{project.name} repository".color(:green) - rescue StandardError => e - progress.puts "[Failed] restoring #{project.name} repository".color(:red) + gitlab_shell.remove_repository(project.repository_storage, project.disk_path) if project.repository_exists? + project.repository.create_from_bundle path_to_project_bundle + restore_repo_status = true + rescue => e + restore_repo_status = false progress.puts "Error: #{e}".color(:red) end else - progress.puts "[Failed] bundle file #{path_to_project_bundle} does not exist" + restore_repo_status = gitlab_shell.create_repository(project.repository_storage, project.disk_path) + end + + if restore_repo_status + progress.puts "[DONE] restoring #{project.name} repository".color(:green) + else + progress.puts "[Failed] restoring #{project.name} repository".color(:red) + end + + gitaly_migrate(:restore_custom_hooks) do |is_enabled| + # TODO: Need to find a way to do this for gitaly + unless is_enabled + in_path(path_to_tars(project)) do |dir| + cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) + + output, status = Gitlab::Popen.popen(cmd) + unless status.zero? + progress_warn(project, cmd.join(' '), output) + end + end + end end wiki = ProjectWiki.new(project) @@ -109,6 +139,7 @@ module Backup if File.exist?(path_to_wiki_bundle) begin + gitlab_shell.remove_repository(project.wiki.repository_storage, project.wiki.disk_path) if project.wiki_repository_exists? project.repository.create_from_bundle(path_to_wiki_bundle) progress.puts "[DONE] restoring #{project.name} wiki".color(:green) rescue StandardError => e @@ -116,6 +147,13 @@ module Backup progress.puts "Error #{e}".color(:red) end end + + gitaly_migrate(:create_hooks) do |is_enabled| + # TODO: Need to find a way to do this for gitaly + unless is_enabled + Gitlab::Git::Repository.create_hooks(path_to_project_repo, Gitlab.config.gitlab_shell.hooks_path) + end + end end end # rubocop:enable Metrics/AbcSize @@ -190,5 +228,13 @@ module Backup def display_repo_path(project) project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path end + + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) + Gitlab::GitalyClient.migrate(method, status: status, &block) + rescue GRPC::NotFound, GRPC::BadStatus => e + # Old Popen code returns [Error, output] to the caller, so we + # need to do the same here... + raise Error, e + end end end -- cgit v1.2.1 From 4ea9458bbeb28d0a9ba70b45810bda9433467ddd Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Fri, 11 May 2018 12:46:42 -0700 Subject: Remove hooks restore that is no longer necessary --- lib/backup/repository.rb | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 74b9e0205a4..02897a11fac 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -147,13 +147,6 @@ module Backup progress.puts "Error #{e}".color(:red) end end - - gitaly_migrate(:create_hooks) do |is_enabled| - # TODO: Need to find a way to do this for gitaly - unless is_enabled - Gitlab::Git::Repository.create_hooks(path_to_project_repo, Gitlab.config.gitlab_shell.hooks_path) - end - end end end # rubocop:enable Metrics/AbcSize -- cgit v1.2.1 From d5bd61e82f425288397b53ba077eb59782fb12ca Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Fri, 11 May 2018 14:07:56 -0700 Subject: Output project name before restoring each project repo --- lib/backup/repository.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 02897a11fac..0e12f70a066 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -96,6 +96,7 @@ module Backup prepare_directories gitlab_shell = Gitlab::Shell.new Project.find_each(batch_size: 1000) do |project| + progress.print " * #{project.name} ... " path_to_project_bundle = path_to_bundle(project) path_to_project_repo = path_to_repo(project) project.ensure_storage_path_exists -- cgit v1.2.1 From b8b82aa50e677d6e38b32080a65b48211350e9af Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Fri, 11 May 2018 15:09:26 -0700 Subject: Fix the paths for wiki restore --- lib/backup/repository.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 0e12f70a066..38751a1040e 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -96,7 +96,7 @@ module Backup prepare_directories gitlab_shell = Gitlab::Shell.new Project.find_each(batch_size: 1000) do |project| - progress.print " * #{project.name} ... " + progress.print " * #{project.full_path} ... " path_to_project_bundle = path_to_bundle(project) path_to_project_repo = path_to_repo(project) project.ensure_storage_path_exists @@ -116,9 +116,9 @@ module Backup end if restore_repo_status - progress.puts "[DONE] restoring #{project.name} repository".color(:green) + progress.puts "[DONE]".color(:green) else - progress.puts "[Failed] restoring #{project.name} repository".color(:red) + progress.puts "[Failed] restoring #{project.full_path} repository".color(:red) end gitaly_migrate(:restore_custom_hooks) do |is_enabled| @@ -139,12 +139,13 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_bundle) + progress.print " * #{wiki.full_path} ... " begin - gitlab_shell.remove_repository(project.wiki.repository_storage, project.wiki.disk_path) if project.wiki_repository_exists? - project.repository.create_from_bundle(path_to_wiki_bundle) - progress.puts "[DONE] restoring #{project.name} wiki".color(:green) + gitlab_shell.remove_repository(wiki.repository_storage, wiki.disk_path) if wiki.repository_exists? + wiki.repository.create_from_bundle(path_to_wiki_bundle) + progress.puts "[DONE]".color(:green) rescue StandardError => e - progress.puts "[Failed] restoring #{project.name} wiki".color(:red) + progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red) progress.puts "Error #{e}".color(:red) end end -- cgit v1.2.1 From 889c62c1e3a2c11e726d1160d92ff83cb3e0fd7d Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Mon, 14 May 2018 13:49:04 -0700 Subject: Link to upstream gitaly issues for the missing restore paths --- lib/backup/repository.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 38751a1040e..ba464368042 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -70,6 +70,7 @@ module Backup Gitlab.config.repositories.storages.each do |name, repository_storage| gitaly_migrate(:remove_repositories) do |is_enabled| # TODO: Need to find a way to do this for gitaly + # Gitaly discussion issue: https://gitlab.com/gitlab-org/gitaly/issues/1194 unless is_enabled path = repository_storage.legacy_disk_path next unless File.exist?(path) @@ -101,21 +102,21 @@ module Backup path_to_project_repo = path_to_repo(project) project.ensure_storage_path_exists - restore_repo_status = nil + restore_repo_success = nil if File.exist?(path_to_project_bundle) begin gitlab_shell.remove_repository(project.repository_storage, project.disk_path) if project.repository_exists? project.repository.create_from_bundle path_to_project_bundle - restore_repo_status = true + restore_repo_success = true rescue => e - restore_repo_status = false + restore_repo_success = false progress.puts "Error: #{e}".color(:red) end else - restore_repo_status = gitlab_shell.create_repository(project.repository_storage, project.disk_path) + restore_repo_success = gitlab_shell.create_repository(project.repository_storage, project.disk_path) end - if restore_repo_status + if restore_repo_success progress.puts "[DONE]".color(:green) else progress.puts "[Failed] restoring #{project.full_path} repository".color(:red) @@ -123,6 +124,7 @@ module Backup gitaly_migrate(:restore_custom_hooks) do |is_enabled| # TODO: Need to find a way to do this for gitaly + # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195 unless is_enabled in_path(path_to_tars(project)) do |dir| cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) @@ -144,7 +146,7 @@ module Backup gitlab_shell.remove_repository(wiki.repository_storage, wiki.disk_path) if wiki.repository_exists? wiki.repository.create_from_bundle(path_to_wiki_bundle) progress.puts "[DONE]".color(:green) - rescue StandardError => e + rescue => e progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red) progress.puts "Error #{e}".color(:red) end -- cgit v1.2.1 From 7ea4a42395eb09f2c2bac2ed74297fca6188fd3b Mon Sep 17 00:00:00 2001 From: Ahmad Hassan Date: Wed, 16 May 2018 17:17:05 +0300 Subject: Fix failing specs --- spec/lib/backup/repository_spec.rb | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/spec/lib/backup/repository_spec.rb b/spec/lib/backup/repository_spec.rb index b1ea9c0b622..3a568506c11 100644 --- a/spec/lib/backup/repository_spec.rb +++ b/spec/lib/backup/repository_spec.rb @@ -49,14 +49,14 @@ describe Backup::Repository do describe 'command failure' do before do - allow(Gitlab::Popen).to receive(:popen).and_return(['error', 1]) + allow_any_instance_of(Gitlab::Shell).to receive(:create_repository).and_return(false) end context 'hashed storage' do it 'shows the appropriate error' do subject.restore - expect(progress).to have_received(:puts).with("Ignoring error on #{project.full_path} (#{project.disk_path}) - error") + expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} repository") end end @@ -66,33 +66,10 @@ describe Backup::Repository do it 'shows the appropriate error' do subject.restore - expect(progress).to have_received(:puts).with("Ignoring error on #{project.full_path} - error") + expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} repository") end end end - - describe 'folders without permissions' do - before do - allow(FileUtils).to receive(:mv).and_raise(Errno::EACCES) - end - - it 'shows error message' do - expect(subject).to receive(:access_denied_error) - subject.restore - end - end - - describe 'folder that is a mountpoint' do - before do - allow(FileUtils).to receive(:mv).and_raise(Errno::EBUSY) - end - - it 'shows error message' do - expect(subject).to receive(:resource_busy_error).and_call_original - - expect { subject.restore }.to raise_error(/is a mountpoint/) - end - end end describe '#empty_repo?' do -- cgit v1.2.1 From 51ee6f0b8ba1995f60299f7f399bed1420de5729 Mon Sep 17 00:00:00 2001 From: Ahmad Hassan Date: Wed, 16 May 2018 20:16:42 +0300 Subject: Changelog entry --- changelogs/unreleased/migrate-restore-repo-to-gitaly.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelogs/unreleased/migrate-restore-repo-to-gitaly.yml diff --git a/changelogs/unreleased/migrate-restore-repo-to-gitaly.yml b/changelogs/unreleased/migrate-restore-repo-to-gitaly.yml new file mode 100644 index 00000000000..59f375de20e --- /dev/null +++ b/changelogs/unreleased/migrate-restore-repo-to-gitaly.yml @@ -0,0 +1,5 @@ +--- +title: Support restoring repositories into gitaly +merge_request: +author: +type: changed -- cgit v1.2.1 From 42cabcb1c45dc8f0d38c36c409bf0efbc9111a67 Mon Sep 17 00:00:00 2001 From: Ahmad Hassan Date: Thu, 17 May 2018 15:34:31 +0300 Subject: Call path_to_repo only when using gitaly --- lib/backup/repository.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index ba464368042..bc517191939 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -99,7 +99,6 @@ module Backup Project.find_each(batch_size: 1000) do |project| progress.print " * #{project.full_path} ... " path_to_project_bundle = path_to_bundle(project) - path_to_project_repo = path_to_repo(project) project.ensure_storage_path_exists restore_repo_success = nil @@ -127,6 +126,7 @@ module Backup # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195 unless is_enabled in_path(path_to_tars(project)) do |dir| + path_to_project_repo = path_to_repo(project) cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) output, status = Gitlab::Popen.popen(cmd) -- cgit v1.2.1 From 62f29887136ba39eb2c0607eba250d488ced2611 Mon Sep 17 00:00:00 2001 From: Ahmad Hassan Date: Thu, 17 May 2018 15:36:58 +0300 Subject: Return from prepare directory if gitaly skip is enabled --- lib/backup/repository.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index bc517191939..2209a305d91 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -67,6 +67,8 @@ module Backup end def prepare_directories + return if GitalyClient.feature_enabled?(:backup_skip_prepare_directories) + Gitlab.config.repositories.storages.each do |name, repository_storage| gitaly_migrate(:remove_repositories) do |is_enabled| # TODO: Need to find a way to do this for gitaly -- cgit v1.2.1 From 125be1865b224c0a34ab8a1bbd62ac33b10e1d9b Mon Sep 17 00:00:00 2001 From: Ahmad Hassan Date: Thu, 17 May 2018 15:39:59 +0300 Subject: Move restore_custom_hooks to its seperate function --- lib/backup/repository.rb | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 2209a305d91..3e7590e55e3 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -67,7 +67,7 @@ module Backup end def prepare_directories - return if GitalyClient.feature_enabled?(:backup_skip_prepare_directories) + return if Gitlab::GitalyClient.feature_enabled?(:backup_skip_prepare_directories) Gitlab.config.repositories.storages.each do |name, repository_storage| gitaly_migrate(:remove_repositories) do |is_enabled| @@ -95,6 +95,24 @@ module Backup end end + def restore_custom_hooks + gitaly_migrate(:restore_custom_hooks) do |is_enabled| + # TODO: Need to find a way to do this for gitaly + # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195 + unless is_enabled + in_path(path_to_tars(project)) do |dir| + path_to_project_repo = path_to_repo(project) + cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) + + output, status = Gitlab::Popen.popen(cmd) + unless status.zero? + progress_warn(project, cmd.join(' '), output) + end + end + end + end + end + def restore prepare_directories gitlab_shell = Gitlab::Shell.new @@ -123,21 +141,7 @@ module Backup progress.puts "[Failed] restoring #{project.full_path} repository".color(:red) end - gitaly_migrate(:restore_custom_hooks) do |is_enabled| - # TODO: Need to find a way to do this for gitaly - # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195 - unless is_enabled - in_path(path_to_tars(project)) do |dir| - path_to_project_repo = path_to_repo(project) - cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) - - output, status = Gitlab::Popen.popen(cmd) - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end - end - end + restore_custom_hooks unless Gitlab::GitalyClient.feature_enabled?(:backup_skip_restore_custom_hooks) wiki = ProjectWiki.new(project) path_to_wiki_bundle = path_to_bundle(wiki) -- cgit v1.2.1 From 80197bdc6212a610f96062b8b4c49771f00c87f3 Mon Sep 17 00:00:00 2001 From: Ahmad Hassan Date: Mon, 21 May 2018 11:45:41 +0200 Subject: Remove gitaly_migrate blocks and replace by early return --- lib/backup/repository.rb | 59 +++++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 3e7590e55e3..a6e9450796f 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -67,48 +67,41 @@ module Backup end def prepare_directories + # TODO: Need to find a way to do this for gitaly + # Gitaly discussion issue: https://gitlab.com/gitlab-org/gitaly/issues/1194 + return if Gitlab::GitalyClient.feature_enabled?(:backup_skip_prepare_directories) Gitlab.config.repositories.storages.each do |name, repository_storage| - gitaly_migrate(:remove_repositories) do |is_enabled| - # TODO: Need to find a way to do this for gitaly - # Gitaly discussion issue: https://gitlab.com/gitlab-org/gitaly/issues/1194 - unless is_enabled - path = repository_storage.legacy_disk_path - next unless File.exist?(path) - - # Move all files in the existing repos directory except . and .. to - # repositories.old. directory - bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) - FileUtils.mkdir_p(bk_repos_path, mode: 0700) - files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] - - begin - FileUtils.mv(files, bk_repos_path) - rescue Errno::EACCES - access_denied_error(path) - rescue Errno::EBUSY - resource_busy_error(path) - end - end + path = repository_storage.legacy_disk_path + next unless File.exist?(path) + + # Move all files in the existing repos directory except . and .. to + # repositories.old. directory + bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) + FileUtils.mkdir_p(bk_repos_path, mode: 0700) + files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] + + begin + FileUtils.mv(files, bk_repos_path) + rescue Errno::EACCES + access_denied_error(path) + rescue Errno::EBUSY + resource_busy_error(path) end end end def restore_custom_hooks - gitaly_migrate(:restore_custom_hooks) do |is_enabled| - # TODO: Need to find a way to do this for gitaly - # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195 - unless is_enabled - in_path(path_to_tars(project)) do |dir| - path_to_project_repo = path_to_repo(project) - cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) + # TODO: Need to find a way to do this for gitaly + # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195 + in_path(path_to_tars(project)) do |dir| + path_to_project_repo = path_to_repo(project) + cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) - output, status = Gitlab::Popen.popen(cmd) - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end + output, status = Gitlab::Popen.popen(cmd) + unless status.zero? + progress_warn(project, cmd.join(' '), output) end end end -- cgit v1.2.1 From 9c4f967785d05b45ce33ece043c6bacb9d798ca1 Mon Sep 17 00:00:00 2001 From: Ahmad Hassan Date: Wed, 23 May 2018 12:31:53 +0200 Subject: Take restoring custom hooks and the skip flag out of scope --- lib/backup/repository.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index a6e9450796f..847d14d01c3 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -70,8 +70,6 @@ module Backup # TODO: Need to find a way to do this for gitaly # Gitaly discussion issue: https://gitlab.com/gitlab-org/gitaly/issues/1194 - return if Gitlab::GitalyClient.feature_enabled?(:backup_skip_prepare_directories) - Gitlab.config.repositories.storages.each do |name, repository_storage| path = repository_storage.legacy_disk_path next unless File.exist?(path) @@ -92,7 +90,7 @@ module Backup end end - def restore_custom_hooks + def restore_custom_hooks(project) # TODO: Need to find a way to do this for gitaly # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195 in_path(path_to_tars(project)) do |dir| @@ -134,7 +132,7 @@ module Backup progress.puts "[Failed] restoring #{project.full_path} repository".color(:red) end - restore_custom_hooks unless Gitlab::GitalyClient.feature_enabled?(:backup_skip_restore_custom_hooks) + restore_custom_hooks(project) wiki = ProjectWiki.new(project) path_to_wiki_bundle = path_to_bundle(wiki) -- cgit v1.2.1 From 84e6ddb41f581064c9bb66c34341ec7248f1b333 Mon Sep 17 00:00:00 2001 From: Ahmad Hassan Date: Thu, 24 May 2018 14:16:31 +0200 Subject: Remove unneeded gitaly_migrate function --- lib/backup/repository.rb | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 847d14d01c3..aa3c3a35764 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -222,13 +222,5 @@ module Backup def display_repo_path(project) project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path end - - def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) - Gitlab::GitalyClient.migrate(method, status: status, &block) - rescue GRPC::NotFound, GRPC::BadStatus => e - # Old Popen code returns [Error, output] to the caller, so we - # need to do the same here... - raise Error, e - end end end -- cgit v1.2.1