diff options
Diffstat (limited to 'lib/backup/repositories.rb')
| -rw-r--r-- | lib/backup/repositories.rb | 194 |
1 files changed, 41 insertions, 153 deletions
diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index b1231eebfcc..80d23c1eb7f 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -4,17 +4,16 @@ require 'yaml' module Backup class Repositories - attr_reader :progress - - def initialize(progress) + def initialize(progress, strategy:) @progress = progress + @strategy = strategy end def dump(max_concurrency:, max_storage_concurrency:) - prepare + strategy.start(:create) if max_concurrency <= 1 && max_storage_concurrency <= 1 - return dump_consecutive + return enqueue_consecutive end check_valid_storages! @@ -25,7 +24,7 @@ module Backup threads = Gitlab.config.repositories.storages.keys.map do |storage| Thread.new do Rails.application.executor.wrap do - dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) + enqueue_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) rescue StandardError => e errors << e end @@ -37,32 +36,24 @@ module Backup end raise errors.pop unless errors.empty? + ensure + strategy.wait end def restore - restore_project_repositories - restore_snippets + strategy.start(:restore) + enqueue_consecutive + + ensure + strategy.wait + cleanup_snippets_without_repositories restore_object_pools end private - def restore_project_repositories - Project.find_each(batch_size: 1000) do |project| - restore_repository(project, Gitlab::GlRepository::PROJECT) - restore_repository(project, Gitlab::GlRepository::WIKI) - restore_repository(project, Gitlab::GlRepository::DESIGN) - end - end - - def restore_snippets - invalid_ids = Snippet.find_each(batch_size: 1000) - .map { |snippet| restore_snippet_repository(snippet) } - .compact - - cleanup_snippets_without_repositories(invalid_ids) - end + attr_reader :progress, :strategy def check_valid_storages! repository_storage_klasses.each do |klass| @@ -76,32 +67,22 @@ module Backup [ProjectRepository, SnippetRepository] end - def backup_repos_path - @backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories') - end - - def prepare - FileUtils.rm_rf(backup_repos_path) - FileUtils.mkdir_p(Gitlab.config.backup.path) - FileUtils.mkdir(backup_repos_path, mode: 0700) + def enqueue_consecutive + enqueue_consecutive_projects + enqueue_consecutive_snippets end - def dump_consecutive - dump_consecutive_projects - dump_consecutive_snippets - end - - def dump_consecutive_projects + def enqueue_consecutive_projects project_relation.find_each(batch_size: 1000) do |project| - dump_project(project) + enqueue_project(project) end end - def dump_consecutive_snippets - Snippet.find_each(batch_size: 1000) { |snippet| dump_snippet(snippet) } + def enqueue_consecutive_snippets + Snippet.find_each(batch_size: 1000) { |snippet| enqueue_snippet(snippet) } end - def dump_storage(storage, semaphore, max_storage_concurrency:) + def enqueue_storage(storage, semaphore, max_storage_concurrency:) errors = Queue.new queue = InterlockSizedQueue.new(1) @@ -114,7 +95,7 @@ module Backup end begin - dump_container(container) + enqueue_container(container) rescue StandardError => e errors << e break @@ -136,23 +117,23 @@ module Backup end end - def dump_container(container) + def enqueue_container(container) case container when Project - dump_project(container) + enqueue_project(container) when Snippet - dump_snippet(container) + enqueue_snippet(container) end end - def dump_project(project) - backup_repository(project, Gitlab::GlRepository::PROJECT) - backup_repository(project, Gitlab::GlRepository::WIKI) - backup_repository(project, Gitlab::GlRepository::DESIGN) + def enqueue_project(project) + strategy.enqueue(project, Gitlab::GlRepository::PROJECT) + strategy.enqueue(project, Gitlab::GlRepository::WIKI) + strategy.enqueue(project, Gitlab::GlRepository::DESIGN) end - def dump_snippet(snippet) - backup_repository(snippet, Gitlab::GlRepository::SNIPPET) + def enqueue_snippet(snippet) + strategy.enqueue(snippet, Gitlab::GlRepository::SNIPPET) end def enqueue_records_for_storage(storage, queue, errors) @@ -181,22 +162,6 @@ module Backup Snippet.id_in(SnippetRepository.for_repository_storage(storage).select(:snippet_id)) end - def backup_repository(container, type) - BackupRestore.new( - progress, - type.repository_for(container), - backup_repos_path - ).backup - end - - def restore_repository(container, type) - BackupRestore.new( - progress, - type.repository_for(container), - backup_repos_path - ).restore(always_create: type.project?) - end - def restore_object_pools PoolRepository.includes(:source_project).find_each do |pool| progress.puts " - Object pool #{pool.disk_path}..." @@ -214,99 +179,22 @@ module Backup end end - def restore_snippet_repository(snippet) - restore_repository(snippet, Gitlab::GlRepository::SNIPPET) - - response = Snippets::RepositoryValidationService.new(nil, snippet).execute - - if response.error? - snippet.repository.remove - - progress.puts("Snippet #{snippet.full_path} can't be restored: #{response.message}") - - snippet.id - else - nil - end - end - # Snippets without a repository should be removed because they failed to import # due to having invalid repositories - def cleanup_snippets_without_repositories(ids) - Snippet.id_in(ids).delete_all - end + def cleanup_snippets_without_repositories + invalid_snippets = [] - class BackupRestore - attr_accessor :progress, :repository, :backup_repos_path + Snippet.find_each(batch_size: 1000).each do |snippet| + response = Snippets::RepositoryValidationService.new(nil, snippet).execute + next if response.success? - def initialize(progress, repository, backup_repos_path) - @progress = progress - @repository = repository - @backup_repos_path = backup_repos_path - end - - def backup - progress.puts " * #{display_repo_path} ... " - - if repository.empty? - progress.puts " * #{display_repo_path} ... " + "[EMPTY] [SKIPPED]".color(:cyan) - return - end - - FileUtils.mkdir_p(repository_backup_path) - - repository.bundle_to_disk(path_to_bundle) - repository.gitaly_repository_client.backup_custom_hooks(custom_hooks_tar) - - progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) - - rescue StandardError => e - progress.puts "[Failed] backing up #{display_repo_path}".color(:red) - progress.puts "Error #{e}".color(:red) - end - - def restore(always_create: false) - progress.puts " * #{display_repo_path} ... " - - repository.remove rescue nil - - if File.exist?(path_to_bundle) - repository.create_from_bundle(path_to_bundle) - restore_custom_hooks - elsif always_create - repository.create_repository - end - - progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) - - rescue StandardError => e - progress.puts "[Failed] restoring #{display_repo_path}".color(:red) - progress.puts "Error #{e}".color(:red) - end - - private - - def display_repo_path - "#{repository.full_path} (#{repository.disk_path})" - end - - def repository_backup_path - @repository_backup_path ||= File.join(backup_repos_path, repository.disk_path) - end - - def path_to_bundle - @path_to_bundle ||= File.join(backup_repos_path, repository.disk_path + '.bundle') - end - - def restore_custom_hooks - return unless File.exist?(custom_hooks_tar) + snippet.repository.remove + progress.puts("Snippet #{snippet.full_path} can't be restored: #{response.message}") - repository.gitaly_repository_client.restore_custom_hooks(custom_hooks_tar) + invalid_snippets << snippet.id end - def custom_hooks_tar - File.join(repository_backup_path, "custom_hooks.tar") - end + Snippet.id_in(invalid_snippets).delete_all end class InterlockSizedQueue < SizedQueue |