Add latest changes from gitlab-org/gitlab@master

6 files changed, 144 insertions, 66 deletions

diff --git a/spec/lib/backup/database_spec.rb b/spec/lib/backup/database_spec.rb
index f57037d5652..4345778ba92 100644
--- a/spec/lib/backup/database_spec.rb
+++ b/spec/lib/backup/database_spec.rb
@@ -6,6 +6,10 @@ RSpec.describe Backup::Database do
   let(:progress) { StringIO.new }
   let(:output) { progress.string }
 
+  before do
+    allow(Gitlab::TaskHelpers).to receive(:ask_to_continue)
+  end
+
   describe '#restore' do
     let(:cmd) { %W[#{Gem.ruby} -e $stdout.puts(1)] }
     let(:data) { Rails.root.join("spec/fixtures/pages_empty.tar.gz").to_s }
@@ -20,7 +24,7 @@ RSpec.describe Backup::Database do
       let(:data) { Rails.root.join("spec/fixtures/pages_empty.tar.gz").to_s }
 
       it 'returns successfully' do
-        expect(subject.restore).to eq([])
+        subject.restore
 
         expect(output).to include("Restoring PostgreSQL database")
         expect(output).to include("[DONE]")
@@ -42,7 +46,8 @@ RSpec.describe Backup::Database do
       let(:cmd) { %W[#{Gem.ruby} -e $stderr.write("#{noise}#{visible_error}")] }
 
       it 'filters out noise from errors' do
-        expect(subject.restore).to eq([visible_error])
+        subject.restore
+
         expect(output).to include("ERRORS")
         expect(output).not_to include(noise)
         expect(output).to include(visible_error)
diff --git a/spec/lib/backup/manager_spec.rb b/spec/lib/backup/manager_spec.rb
index 9c186205067..ac693ad8b98 100644
--- a/spec/lib/backup/manager_spec.rb
+++ b/spec/lib/backup/manager_spec.rb
@@ -12,6 +12,11 @@ RSpec.describe Backup::Manager do
   before do
     allow(progress).to receive(:puts)
     allow(progress).to receive(:print)
+    FileUtils.mkdir_p('tmp/tests/public/uploads')
+  end
+
+  after do
+    FileUtils.rm_rf('tmp/tests/public/uploads', secure: true)
   end
 
   describe '#pack' do
diff --git a/spec/lib/backup/repositories_spec.rb b/spec/lib/backup/repositories_spec.rb
index b06f2237211..0b29a25360d 100644
--- a/spec/lib/backup/repositories_spec.rb
+++ b/spec/lib/backup/repositories_spec.rb
@@ -6,8 +6,17 @@ RSpec.describe Backup::Repositories do
   let(:progress) { spy(:stdout) }
   let(:parallel_enqueue) { true }
   let(:strategy) { spy(:strategy, parallel_enqueue?: parallel_enqueue) }
-
-  subject { described_class.new(progress, strategy: strategy) }
+  let(:max_concurrency) { 1 }
+  let(:max_storage_concurrency) { 1 }
+
+  subject do
+    described_class.new(
+      progress,
+      strategy: strategy,
+      max_concurrency: max_concurrency,
+      max_storage_concurrency: max_storage_concurrency
+    )
+  end
 
   describe '#dump' do
     let_it_be(:projects) { create_list(:project, 5, :repository) }
@@ -17,7 +26,7 @@ RSpec.describe Backup::Repositories do
         project_snippet = create(:project_snippet, :repository, project: project)
         personal_snippet = create(:personal_snippet, :repository, author: project.first_owner)
 
-        subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+        subject.dump
 
         expect(strategy).to have_received(:start).with(:create)
         expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
@@ -51,38 +60,40 @@ RSpec.describe Backup::Repositories do
         end
         expect(strategy).to receive(:finish!)
 
-        subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+        subject.dump
       end
 
       describe 'command failure' do
         it 'enqueue_project raises an error' do
           allow(strategy).to receive(:enqueue).with(anything, Gitlab::GlRepository::PROJECT).and_raise(IOError)
 
-          expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(IOError)
+          expect { subject.dump }.to raise_error(IOError)
         end
 
         it 'project query raises an error' do
           allow(Project).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
 
-          expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(ActiveRecord::StatementTimeout)
+          expect { subject.dump }.to raise_error(ActiveRecord::StatementTimeout)
         end
       end
 
       it 'avoids N+1 database queries' do
         control_count = ActiveRecord::QueryRecorder.new do
-          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+          subject.dump
         end.count
 
         create_list(:project, 2, :repository)
 
         expect do
-          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+          subject.dump
         end.not_to exceed_query_limit(control_count)
       end
     end
 
     context 'concurrency with a strategy without parallel enqueueing support' do
       let(:parallel_enqueue) { false }
+      let(:max_concurrency) { 2 }
+      let(:max_storage_concurrency) { 2 }
 
       it 'enqueues all projects sequentially' do
         expect(Thread).not_to receive(:new)
@@ -93,13 +104,14 @@ RSpec.describe Backup::Repositories do
         end
         expect(strategy).to receive(:finish!)
 
-        subject.dump(max_concurrency: 2, max_storage_concurrency: 2)
+        subject.dump
       end
     end
 
     [4, 10].each do |max_storage_concurrency|
       context "max_storage_concurrency #{max_storage_concurrency}", quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/241701' do
         let(:storage_keys) { %w[default test_second_storage] }
+        let(:max_storage_concurrency) { max_storage_concurrency }
 
         before do
           allow(Gitlab.config.repositories.storages).to receive(:keys).and_return(storage_keys)
@@ -116,54 +128,58 @@ RSpec.describe Backup::Repositories do
           end
           expect(strategy).to receive(:finish!)
 
-          subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency)
+          subject.dump
         end
 
-        it 'creates the expected number of threads with extra max concurrency' do
-          expect(Thread).to receive(:new)
-            .exactly(storage_keys.length * (max_storage_concurrency + 1)).times
-            .and_call_original
+        context 'with extra max concurrency' do
+          let(:max_concurrency) { 3 }
 
-          expect(strategy).to receive(:start).with(:create)
-          projects.each do |project|
-            expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
-          end
-          expect(strategy).to receive(:finish!)
+          it 'creates the expected number of threads' do
+            expect(Thread).to receive(:new)
+              .exactly(storage_keys.length * (max_storage_concurrency + 1)).times
+              .and_call_original
 
-          subject.dump(max_concurrency: 3, max_storage_concurrency: max_storage_concurrency)
+            expect(strategy).to receive(:start).with(:create)
+            projects.each do |project|
+              expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
+            end
+            expect(strategy).to receive(:finish!)
+
+            subject.dump
+          end
         end
 
         describe 'command failure' do
           it 'enqueue_project raises an error' do
             allow(strategy).to receive(:enqueue).and_raise(IOError)
 
-            expect { subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) }.to raise_error(IOError)
+            expect { subject.dump }.to raise_error(IOError)
           end
 
           it 'project query raises an error' do
             allow(Project).to receive_message_chain(:for_repository_storage, :includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
 
-            expect { subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) }.to raise_error(ActiveRecord::StatementTimeout)
+            expect { subject.dump }.to raise_error(ActiveRecord::StatementTimeout)
           end
 
           context 'misconfigured storages' do
             let(:storage_keys) { %w[test_second_storage] }
 
             it 'raises an error' do
-              expect { subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) }.to raise_error(Backup::Error, 'repositories.storages in gitlab.yml is misconfigured')
+              expect { subject.dump }.to raise_error(Backup::Error, 'repositories.storages in gitlab.yml is misconfigured')
             end
           end
         end
 
         it 'avoids N+1 database queries' do
           control_count = ActiveRecord::QueryRecorder.new do
-            subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency)
+            subject.dump
           end.count
 
           create_list(:project, 2, :repository)
 
           expect do
-            subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency)
+            subject.dump
           end.not_to exceed_query_limit(control_count)
         end
       end
diff --git a/spec/migrations/20220211214605_update_integrations_trigger_type_new_on_insert_null_safe_spec.rb b/spec/migrations/20220211214605_update_integrations_trigger_type_new_on_insert_null_safe_spec.rb
new file mode 100644
index 00000000000..bf79ee02ff1
--- /dev/null
+++ b/spec/migrations/20220211214605_update_integrations_trigger_type_new_on_insert_null_safe_spec.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require_migration!
+
+RSpec.describe UpdateIntegrationsTriggerTypeNewOnInsertNullSafe, :migration do
+  let(:integrations) { table(:integrations) }
+
+  before do
+    migrate!
+  end
+
+  it 'leaves defined values alone' do
+    record = integrations.create!(type: 'XService', type_new: 'Integrations::Y')
+
+    expect(integrations.find(record.id)).to have_attributes(type: 'XService', type_new: 'Integrations::Y')
+  end
+
+  it 'keeps type_new synchronized with type' do
+    record = integrations.create!(type: 'AbcService', type_new: nil)
+
+    expect(integrations.find(record.id)).to have_attributes(
+      type: 'AbcService',
+      type_new: 'Integrations::Abc'
+    )
+  end
+
+  it 'keeps type synchronized with type_new' do
+    record = integrations.create!(type: nil, type_new: 'Integrations::Abc')
+
+    expect(integrations.find(record.id)).to have_attributes(
+      type: 'AbcService',
+      type_new: 'Integrations::Abc'
+    )
+  end
+end
diff --git a/spec/services/ci/copy_cross_database_associations_service_spec.rb b/spec/services/ci/copy_cross_database_associations_service_spec.rb
new file mode 100644
index 00000000000..5938ac258d0
--- /dev/null
+++ b/spec/services/ci/copy_cross_database_associations_service_spec.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::CopyCrossDatabaseAssociationsService do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:pipeline) { create(:ci_pipeline, project: project) }
+  let_it_be(:old_build) { create(:ci_build, pipeline: pipeline) }
+  let_it_be(:new_build) { create(:ci_build, pipeline: pipeline) }
+
+  subject(:execute) { described_class.new.execute(old_build, new_build) }
+
+  describe '#execute' do
+    it 'returns a success response' do
+      expect(execute).to be_success
+    end
+  end
+end
diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb
index 92843f48b8b..e9aa8cbb991 100644
--- a/spec/tasks/gitlab/backup_rake_spec.rb
+++ b/spec/tasks/gitlab/backup_rake_spec.rb
@@ -4,7 +4,8 @@ require 'rake_helper'
 
 RSpec.describe 'gitlab:app namespace rake task', :delete do
   let(:enable_registry) { true }
-  let(:backup_types) { %w{db repo uploads builds artifacts pages lfs terraform_state registry packages} }
+  let(:backup_tasks) { %w{db repo uploads builds artifacts pages lfs terraform_state registry packages} }
+  let(:backup_types) { %w{db repositories uploads builds artifacts pages lfs terraform_state registry packages} }
 
   def tars_glob
     Dir.glob(File.join(Gitlab.config.backup.path, '*_gitlab_backup.tar'))
@@ -48,7 +49,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
   end
 
   def reenable_backup_sub_tasks
-    backup_types.each do |subtask|
+    backup_tasks.each do |subtask|
       Rake::Task["gitlab:backup:#{subtask}:create"].reenable
     end
   end
@@ -72,8 +73,11 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           allow(YAML).to receive(:load_file)
             .and_return({ gitlab_version: gitlab_version })
           expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
-          backup_types.each do |subtask|
-            expect(Rake::Task["gitlab:backup:#{subtask}:restore"]).to receive(:invoke)
+          expect_next_instance_of(::Backup::Manager) do |instance|
+            backup_types.each do |subtask|
+              expect(instance).to receive(:run_restore_task).with(subtask).ordered
+            end
+            expect(instance).not_to receive(:run_restore_task)
           end
           expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
         end
@@ -128,16 +132,14 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           .and_return({ gitlab_version: Gitlab::VERSION })
 
         expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:db:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:repo:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:builds:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:uploads:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:pages:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:lfs:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:terraform_state:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:registry:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:packages:restore']).to receive(:invoke)
+
+        expect_next_instance_of(::Backup::Manager) do |instance|
+          backup_types.each do |subtask|
+            expect(instance).to receive(:run_restore_task).with(subtask).ordered
+          end
+          expect(instance).not_to receive(:run_restore_task)
+        end
+
         expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
       end
 
@@ -198,7 +200,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
 
       context 'specific backup tasks' do
         it 'prints a progress message to stdout' do
-          backup_types.each do |task|
+          backup_tasks.each do |task|
             expect { run_rake_task("gitlab:backup:#{task}:create") }.to output(/Dumping /).to_stdout_from_any_process
           end
         end
@@ -206,7 +208,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         it 'logs the progress to log file' do
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping database ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "[SKIPPED]")
-          expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping repositories ...")
+          expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping repositories ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping uploads ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping builds ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping artifacts ... ")
@@ -217,7 +219,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping packages ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "done").exactly(9).times
 
-          backup_types.each do |task|
+          backup_tasks.each do |task|
             run_rake_task("gitlab:backup:#{task}:create")
           end
         end
@@ -414,11 +416,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       end
 
       it 'has defaults' do
-        expect_next_instance_of(::Backup::Repositories) do |instance|
-          expect(instance).to receive(:dump)
-            .with(max_concurrency: 1, max_storage_concurrency: 1)
-            .and_call_original
-        end
+        expect(::Backup::Repositories).to receive(:new)
+          .with(anything, strategy: anything, max_concurrency: 1, max_storage_concurrency: 1)
+          .and_call_original
 
         expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
       end
@@ -432,11 +432,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         stub_env('GITLAB_BACKUP_MAX_CONCURRENCY', 5)
         stub_env('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 2)
 
-        expect_next_instance_of(::Backup::Repositories) do |instance|
-          expect(instance).to receive(:dump)
-            .with(max_concurrency: 5, max_storage_concurrency: 2)
-            .and_call_original
-        end
+        expect(::Backup::Repositories).to receive(:new)
+          .with(anything, strategy: anything, max_concurrency: 5, max_storage_concurrency: 2)
+          .and_call_original
         expect(::Backup::GitalyBackup).to receive(:new).with(anything, max_parallelism: 5, storage_parallelism: 2).and_call_original
 
         expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
@@ -489,16 +487,12 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         .to receive(:invoke).and_return(true)
 
       expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:db:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:repo:restore']).not_to receive :invoke
-      expect(Rake::Task['gitlab:backup:uploads:restore']).not_to receive :invoke
-      expect(Rake::Task['gitlab:backup:builds:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:pages:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:lfs:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:terraform_state:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:registry:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:packages:restore']).to receive :invoke
+      expect_next_instance_of(::Backup::Manager) do |instance|
+        (backup_types - %w{repositories uploads}).each do |subtask|
+          expect(instance).to receive(:run_restore_task).with(subtask).ordered
+        end
+        expect(instance).not_to receive(:run_restore_task)
+      end
       expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
       expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
     end
@@ -538,8 +532,11 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         .to receive(:invoke).and_return(true)
 
       expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
-      backup_types.each do |subtask|
-        expect(Rake::Task["gitlab:backup:#{subtask}:restore"]).to receive :invoke
+      expect_next_instance_of(::Backup::Manager) do |instance|
+        backup_types.each do |subtask|
+          expect(instance).to receive(:run_restore_task).with(subtask).ordered
+        end
+        expect(instance).not_to receive(:run_restore_task)
       end
       expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
       expect { run_rake_task("gitlab:backup:restore") }.to output.to_stdout_from_any_process