Add latest changes from gitlab-org/gitlab@14-0-stable-eev14.0.0-rc42

3 files changed, 236 insertions, 153 deletions

diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb
new file mode 100644
index 00000000000..cfd3d463f9e
--- /dev/null
+++ b/lib/backup/gitaly_backup.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Backup
+  # Backup and restores repositories using gitaly-backup
+  class GitalyBackup
+    def initialize(progress)
+      @progress = progress
+    end
+
+    def start(type)
+      raise Error, 'already started' if started?
+
+      command = case type
+                when :create
+                  'create'
+                when :restore
+                  'restore'
+                else
+                  raise Error, "unknown backup type: #{type}"
+                end
+
+      @read_io, @write_io = IO.pipe
+      @pid = Process.spawn(bin_path, command, '-path', backup_repos_path, in: @read_io, out: progress)
+    end
+
+    def wait
+      return unless started?
+
+      @write_io.close
+      Process.wait(@pid)
+      status = $?
+
+      @pid = nil
+
+      raise Error, "gitaly-backup exit status #{status.exitstatus}" if status.exitstatus != 0
+    end
+
+    def enqueue(container, repo_type)
+      raise Error, 'not started' unless started?
+
+      repository = repo_type.repository_for(container)
+
+      @write_io.puts({
+        storage_name: repository.storage,
+        relative_path: repository.relative_path,
+        gl_project_path: repository.gl_project_path,
+        always_create: repo_type.project?
+      }.merge(Gitlab::GitalyClient.connection_data(repository.storage)).to_json)
+    end
+
+    private
+
+    attr_reader :progress
+
+    def started?
+      @pid.present?
+    end
+
+    def backup_repos_path
+      File.absolute_path(File.join(Gitlab.config.backup.path, 'repositories'))
+    end
+
+    def bin_path
+      File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-backup'))
+    end
+  end
+end
diff --git a/lib/backup/gitaly_rpc_backup.rb b/lib/backup/gitaly_rpc_backup.rb
new file mode 100644
index 00000000000..53f1de40509
--- /dev/null
+++ b/lib/backup/gitaly_rpc_backup.rb
@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Backup
+  # Backup and restores repositories using the gitaly RPC
+  class GitalyRpcBackup
+    def initialize(progress)
+      @progress = progress
+    end
+
+    def start(type)
+      raise Error, 'already started' if @type
+
+      @type = type
+      case type
+      when :create
+        FileUtils.rm_rf(backup_repos_path)
+        FileUtils.mkdir_p(Gitlab.config.backup.path)
+        FileUtils.mkdir(backup_repos_path, mode: 0700)
+      when :restore
+        # no op
+      else
+        raise Error, "unknown backup type: #{type}"
+      end
+    end
+
+    def wait
+      @type = nil
+    end
+
+    def enqueue(container, repository_type)
+      backup_restore = BackupRestore.new(
+        progress,
+        repository_type.repository_for(container),
+        backup_repos_path
+      )
+
+      case @type
+      when :create
+        backup_restore.backup
+      when :restore
+        backup_restore.restore(always_create: repository_type.project?)
+      else
+        raise Error, 'not started'
+      end
+    end
+
+    private
+
+    attr_reader :progress
+
+    def backup_repos_path
+      @backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories')
+    end
+
+    class BackupRestore
+      attr_accessor :progress, :repository, :backup_repos_path
+
+      def initialize(progress, repository, backup_repos_path)
+        @progress = progress
+        @repository = repository
+        @backup_repos_path = backup_repos_path
+      end
+
+      def backup
+        progress.puts " * #{display_repo_path} ... "
+
+        if repository.empty?
+          progress.puts " * #{display_repo_path} ... " + "[EMPTY] [SKIPPED]".color(:cyan)
+          return
+        end
+
+        FileUtils.mkdir_p(repository_backup_path)
+
+        repository.bundle_to_disk(path_to_bundle)
+        repository.gitaly_repository_client.backup_custom_hooks(custom_hooks_tar)
+
+        progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green)
+
+      rescue StandardError => e
+        progress.puts "[Failed] backing up #{display_repo_path}".color(:red)
+        progress.puts "Error #{e}".color(:red)
+      end
+
+      def restore(always_create: false)
+        progress.puts " * #{display_repo_path} ... "
+
+        repository.remove rescue nil
+
+        if File.exist?(path_to_bundle)
+          repository.create_from_bundle(path_to_bundle)
+          restore_custom_hooks
+        elsif always_create
+          repository.create_repository
+        end
+
+        progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green)
+
+      rescue StandardError => e
+        progress.puts "[Failed] restoring #{display_repo_path}".color(:red)
+        progress.puts "Error #{e}".color(:red)
+      end
+
+      private
+
+      def display_repo_path
+        "#{repository.full_path} (#{repository.disk_path})"
+      end
+
+      def repository_backup_path
+        @repository_backup_path ||= File.join(backup_repos_path, repository.disk_path)
+      end
+
+      def path_to_bundle
+        @path_to_bundle ||= File.join(backup_repos_path, repository.disk_path + '.bundle')
+      end
+
+      def restore_custom_hooks
+        return unless File.exist?(custom_hooks_tar)
+
+        repository.gitaly_repository_client.restore_custom_hooks(custom_hooks_tar)
+      end
+
+      def custom_hooks_tar
+        File.join(repository_backup_path, "custom_hooks.tar")
+      end
+    end
+  end
+end
diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb
index b1231eebfcc..80d23c1eb7f 100644
--- a/lib/backup/repositories.rb
+++ b/lib/backup/repositories.rb
@@ -4,17 +4,16 @@ require 'yaml'
 
 module Backup
   class Repositories
-    attr_reader :progress
-
-    def initialize(progress)
+    def initialize(progress, strategy:)
       @progress = progress
+      @strategy = strategy
     end
 
     def dump(max_concurrency:, max_storage_concurrency:)
-      prepare
+      strategy.start(:create)
 
       if max_concurrency <= 1 && max_storage_concurrency <= 1
-        return dump_consecutive
+        return enqueue_consecutive
       end
 
       check_valid_storages!
@@ -25,7 +24,7 @@ module Backup
       threads = Gitlab.config.repositories.storages.keys.map do |storage|
         Thread.new do
           Rails.application.executor.wrap do
-            dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency)
+            enqueue_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency)
           rescue StandardError => e
             errors << e
           end
@@ -37,32 +36,24 @@ module Backup
       end
 
       raise errors.pop unless errors.empty?
+    ensure
+      strategy.wait
     end
 
     def restore
-      restore_project_repositories
-      restore_snippets
+      strategy.start(:restore)
+      enqueue_consecutive
+
+    ensure
+      strategy.wait
 
+      cleanup_snippets_without_repositories
       restore_object_pools
     end
 
     private
 
-    def restore_project_repositories
-      Project.find_each(batch_size: 1000) do |project|
-        restore_repository(project, Gitlab::GlRepository::PROJECT)
-        restore_repository(project, Gitlab::GlRepository::WIKI)
-        restore_repository(project, Gitlab::GlRepository::DESIGN)
-      end
-    end
-
-    def restore_snippets
-      invalid_ids = Snippet.find_each(batch_size: 1000)
-        .map { |snippet| restore_snippet_repository(snippet) }
-        .compact
-
-      cleanup_snippets_without_repositories(invalid_ids)
-    end
+    attr_reader :progress, :strategy
 
     def check_valid_storages!
       repository_storage_klasses.each do |klass|
@@ -76,32 +67,22 @@ module Backup
       [ProjectRepository, SnippetRepository]
     end
 
-    def backup_repos_path
-      @backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories')
-    end
-
-    def prepare
-      FileUtils.rm_rf(backup_repos_path)
-      FileUtils.mkdir_p(Gitlab.config.backup.path)
-      FileUtils.mkdir(backup_repos_path, mode: 0700)
+    def enqueue_consecutive
+      enqueue_consecutive_projects
+      enqueue_consecutive_snippets
     end
 
-    def dump_consecutive
-      dump_consecutive_projects
-      dump_consecutive_snippets
-    end
-
-    def dump_consecutive_projects
+    def enqueue_consecutive_projects
       project_relation.find_each(batch_size: 1000) do |project|
-        dump_project(project)
+        enqueue_project(project)
       end
     end
 
-    def dump_consecutive_snippets
-      Snippet.find_each(batch_size: 1000) { |snippet| dump_snippet(snippet) }
+    def enqueue_consecutive_snippets
+      Snippet.find_each(batch_size: 1000) { |snippet| enqueue_snippet(snippet) }
     end
 
-    def dump_storage(storage, semaphore, max_storage_concurrency:)
+    def enqueue_storage(storage, semaphore, max_storage_concurrency:)
       errors = Queue.new
       queue = InterlockSizedQueue.new(1)
 
@@ -114,7 +95,7 @@ module Backup
               end
 
               begin
-                dump_container(container)
+                enqueue_container(container)
               rescue StandardError => e
                 errors << e
                 break
@@ -136,23 +117,23 @@ module Backup
       end
     end
 
-    def dump_container(container)
+    def enqueue_container(container)
       case container
       when Project
-        dump_project(container)
+        enqueue_project(container)
       when Snippet
-        dump_snippet(container)
+        enqueue_snippet(container)
       end
     end
 
-    def dump_project(project)
-      backup_repository(project, Gitlab::GlRepository::PROJECT)
-      backup_repository(project, Gitlab::GlRepository::WIKI)
-      backup_repository(project, Gitlab::GlRepository::DESIGN)
+    def enqueue_project(project)
+      strategy.enqueue(project, Gitlab::GlRepository::PROJECT)
+      strategy.enqueue(project, Gitlab::GlRepository::WIKI)
+      strategy.enqueue(project, Gitlab::GlRepository::DESIGN)
     end
 
-    def dump_snippet(snippet)
-      backup_repository(snippet, Gitlab::GlRepository::SNIPPET)
+    def enqueue_snippet(snippet)
+      strategy.enqueue(snippet, Gitlab::GlRepository::SNIPPET)
     end
 
     def enqueue_records_for_storage(storage, queue, errors)
@@ -181,22 +162,6 @@ module Backup
       Snippet.id_in(SnippetRepository.for_repository_storage(storage).select(:snippet_id))
     end
 
-    def backup_repository(container, type)
-      BackupRestore.new(
-        progress,
-        type.repository_for(container),
-        backup_repos_path
-      ).backup
-    end
-
-    def restore_repository(container, type)
-      BackupRestore.new(
-        progress,
-        type.repository_for(container),
-        backup_repos_path
-      ).restore(always_create: type.project?)
-    end
-
     def restore_object_pools
       PoolRepository.includes(:source_project).find_each do |pool|
         progress.puts " - Object pool #{pool.disk_path}..."
@@ -214,99 +179,22 @@ module Backup
       end
     end
 
-    def restore_snippet_repository(snippet)
-      restore_repository(snippet, Gitlab::GlRepository::SNIPPET)
-
-      response = Snippets::RepositoryValidationService.new(nil, snippet).execute
-
-      if response.error?
-        snippet.repository.remove
-
-        progress.puts("Snippet #{snippet.full_path} can't be restored: #{response.message}")
-
-        snippet.id
-      else
-        nil
-      end
-    end
-
     # Snippets without a repository should be removed because they failed to import
     # due to having invalid repositories
-    def cleanup_snippets_without_repositories(ids)
-      Snippet.id_in(ids).delete_all
-    end
+    def cleanup_snippets_without_repositories
+      invalid_snippets = []
 
-    class BackupRestore
-      attr_accessor :progress, :repository, :backup_repos_path
+      Snippet.find_each(batch_size: 1000).each do |snippet|
+        response = Snippets::RepositoryValidationService.new(nil, snippet).execute
+        next if response.success?
 
-      def initialize(progress, repository, backup_repos_path)
-        @progress = progress
-        @repository = repository
-        @backup_repos_path = backup_repos_path
-      end
-
-      def backup
-        progress.puts " * #{display_repo_path} ... "
-
-        if repository.empty?
-          progress.puts " * #{display_repo_path} ... " + "[EMPTY] [SKIPPED]".color(:cyan)
-          return
-        end
-
-        FileUtils.mkdir_p(repository_backup_path)
-
-        repository.bundle_to_disk(path_to_bundle)
-        repository.gitaly_repository_client.backup_custom_hooks(custom_hooks_tar)
-
-        progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green)
-
-      rescue StandardError => e
-        progress.puts "[Failed] backing up #{display_repo_path}".color(:red)
-        progress.puts "Error #{e}".color(:red)
-      end
-
-      def restore(always_create: false)
-        progress.puts " * #{display_repo_path} ... "
-
-        repository.remove rescue nil
-
-        if File.exist?(path_to_bundle)
-          repository.create_from_bundle(path_to_bundle)
-          restore_custom_hooks
-        elsif always_create
-          repository.create_repository
-        end
-
-        progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green)
-
-      rescue StandardError => e
-        progress.puts "[Failed] restoring #{display_repo_path}".color(:red)
-        progress.puts "Error #{e}".color(:red)
-      end
-
-      private
-
-      def display_repo_path
-        "#{repository.full_path} (#{repository.disk_path})"
-      end
-
-      def repository_backup_path
-        @repository_backup_path ||= File.join(backup_repos_path, repository.disk_path)
-      end
-
-      def path_to_bundle
-        @path_to_bundle ||= File.join(backup_repos_path, repository.disk_path + '.bundle')
-      end
-
-      def restore_custom_hooks
-        return unless File.exist?(custom_hooks_tar)
+        snippet.repository.remove
+        progress.puts("Snippet #{snippet.full_path} can't be restored: #{response.message}")
 
-        repository.gitaly_repository_client.restore_custom_hooks(custom_hooks_tar)
+        invalid_snippets << snippet.id
       end
 
-      def custom_hooks_tar
-        File.join(repository_backup_path, "custom_hooks.tar")
-      end
+      Snippet.id_in(invalid_snippets).delete_all
     end
 
     class InterlockSizedQueue < SizedQueue