Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42

4 files changed, 44 insertions, 23 deletions

diff --git a/lib/backup/database.rb b/lib/backup/database.rb
index 0429d9496d6..f07fd786b4b 100644
--- a/lib/backup/database.rb
+++ b/lib/backup/database.rb
@@ -9,6 +9,8 @@ module Backup
     attr_reader :config, :db_file_name
 
     IGNORED_ERRORS = [
+      # Ignore warnings
+      /WARNING:/,
       # Ignore the DROP errors; recent database dumps will use --if-exists with pg_dump
       /does not exist$/,
       # User may not have permissions to drop extensions or schemas
@@ -18,7 +20,7 @@ module Backup
 
     def initialize(progress, filename: nil)
       @progress = progress
-      @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
+      @config = ActiveRecord::Base.configurations.find_db_config(Rails.env).configuration_hash
       @db_file_name = filename || File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
     end
 
@@ -30,9 +32,9 @@ module Backup
       compress_rd.close
 
       dump_pid =
-        case config["adapter"]
+        case config[:adapter]
         when "postgresql" then
-          progress.print "Dumping PostgreSQL database #{config['database']} ... "
+          progress.print "Dumping PostgreSQL database #{database} ... "
           pg_env
           pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump.
           pgsql_args << '--if-exists'
@@ -47,7 +49,7 @@ module Backup
             end
           end
 
-          Process.spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr)
+          Process.spawn('pg_dump', *pgsql_args, database, out: compress_wr)
         end
       compress_wr.close
 
@@ -68,9 +70,9 @@ module Backup
       decompress_wr.close
 
       status, errors =
-        case config["adapter"]
+        case config[:adapter]
         when "postgresql" then
-          progress.print "Restoring PostgreSQL database #{config['database']} ... "
+          progress.print "Restoring PostgreSQL database #{database} ... "
           pg_env
           execute_and_track_errors(pg_restore_cmd, decompress_rd)
         end
@@ -93,6 +95,10 @@ module Backup
 
     protected
 
+    def database
+      @config[:database]
+    end
+
     def ignore_error?(line)
       IGNORED_ERRORS_REGEXP.match?(line)
     end
@@ -128,17 +134,17 @@ module Backup
 
     def pg_env
       args = {
-        'username'  => 'PGUSER',
-        'host'      => 'PGHOST',
-        'port'      => 'PGPORT',
-        'password'  => 'PGPASSWORD',
+        username: 'PGUSER',
+        host: 'PGHOST',
+        port: 'PGPORT',
+        password: 'PGPASSWORD',
         # SSL
-        'sslmode'         => 'PGSSLMODE',
-        'sslkey'          => 'PGSSLKEY',
-        'sslcert'         => 'PGSSLCERT',
-        'sslrootcert'     => 'PGSSLROOTCERT',
-        'sslcrl'          => 'PGSSLCRL',
-        'sslcompression'  => 'PGSSLCOMPRESSION'
+        sslmode: 'PGSSLMODE',
+        sslkey: 'PGSSLKEY',
+        sslcert: 'PGSSLCERT',
+        sslrootcert: 'PGSSLROOTCERT',
+        sslcrl: 'PGSSLCRL',
+        sslcompression: 'PGSSLCOMPRESSION'
       }
       args.each do |opt, arg|
         # This enables the use of different PostgreSQL settings in
@@ -161,7 +167,7 @@ module Backup
     private
 
     def pg_restore_cmd
-      ['psql', config['database']]
+      ['psql', database]
     end
   end
 end
diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb
index cfd3d463f9e..c15b0ed6a1b 100644
--- a/lib/backup/gitaly_backup.rb
+++ b/lib/backup/gitaly_backup.rb
@@ -3,8 +3,10 @@
 module Backup
   # Backup and restores repositories using gitaly-backup
   class GitalyBackup
-    def initialize(progress)
+    def initialize(progress, parallel: nil, parallel_storage: nil)
       @progress = progress
+      @parallel = parallel
+      @parallel_storage = parallel_storage
     end
 
     def start(type)
@@ -19,8 +21,12 @@ module Backup
                   raise Error, "unknown backup type: #{type}"
                 end
 
+      args = []
+      args += ['-parallel', @parallel.to_s] if type == :create && @parallel
+      args += ['-parallel-storage', @parallel_storage.to_s] if type == :create && @parallel_storage
+
       @read_io, @write_io = IO.pipe
-      @pid = Process.spawn(bin_path, command, '-path', backup_repos_path, in: @read_io, out: progress)
+      @pid = Process.spawn(bin_path, command, '-path', backup_repos_path, *args, in: @read_io, out: @progress)
     end
 
     def wait
@@ -48,9 +54,11 @@ module Backup
       }.merge(Gitlab::GitalyClient.connection_data(repository.storage)).to_json)
     end
 
-    private
+    def parallel_enqueue?
+      false
+    end
 
-    attr_reader :progress
+    private
 
     def started?
       @pid.present?
@@ -61,7 +69,7 @@ module Backup
     end
 
     def bin_path
-      File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-backup'))
+      File.absolute_path(Gitlab.config.backup.gitaly_backup_path)
     end
   end
 end
diff --git a/lib/backup/gitaly_rpc_backup.rb b/lib/backup/gitaly_rpc_backup.rb
index 53f1de40509..baac4eb26ca 100644
--- a/lib/backup/gitaly_rpc_backup.rb
+++ b/lib/backup/gitaly_rpc_backup.rb
@@ -44,6 +44,10 @@ module Backup
       end
     end
 
+    def parallel_enqueue?
+      true
+    end
+
     private
 
     attr_reader :progress
diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb
index 80d23c1eb7f..0b5a62529b4 100644
--- a/lib/backup/repositories.rb
+++ b/lib/backup/repositories.rb
@@ -12,7 +12,10 @@ module Backup
     def dump(max_concurrency:, max_storage_concurrency:)
       strategy.start(:create)
 
-      if max_concurrency <= 1 && max_storage_concurrency <= 1
+      # gitaly-backup is designed to handle concurrency on its own. So we want
+      # to avoid entering the buggy concurrency code here when gitaly-backup
+      # is enabled.
+      if (max_concurrency <= 1 && max_storage_concurrency <= 1) || !strategy.parallel_enqueue?
         return enqueue_consecutive
       end