Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42

4 files changed, 77 insertions, 44 deletions

diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb
index 99deaa8d154..c5e73aa3b45 100644
--- a/spec/tasks/gitlab/backup_rake_spec.rb
+++ b/spec/tasks/gitlab/backup_rake_spec.rb
@@ -4,6 +4,7 @@ require 'rake_helper'
 
 RSpec.describe 'gitlab:app namespace rake task', :delete do
   let(:enable_registry) { true }
+  let(:backup_types) { %w{db repo uploads builds artifacts pages lfs terraform_state registry packages} }
 
   def tars_glob
     Dir.glob(File.join(Gitlab.config.backup.path, '*_gitlab_backup.tar'))
@@ -14,7 +15,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
   end
 
   def backup_files
-    %w(backup_information.yml artifacts.tar.gz builds.tar.gz lfs.tar.gz pages.tar.gz)
+    %w(backup_information.yml artifacts.tar.gz builds.tar.gz lfs.tar.gz terraform_state.tar.gz pages.tar.gz packages.tar.gz)
   end
 
   def backup_directories
@@ -47,7 +48,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
   end
 
   def reenable_backup_sub_tasks
-    %w{db repo uploads builds artifacts pages lfs registry}.each do |subtask|
+    backup_types.each do |subtask|
       Rake::Task["gitlab:backup:#{subtask}:create"].reenable
     end
   end
@@ -71,14 +72,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           allow(YAML).to receive(:load_file)
             .and_return({ gitlab_version: gitlab_version })
           expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:db:restore']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:repo:restore']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:builds:restore']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:uploads:restore']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:pages:restore']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:lfs:restore']).to receive(:invoke)
-          expect(Rake::Task['gitlab:backup:registry:restore']).to receive(:invoke)
+          backup_types.each do |subtask|
+            expect(Rake::Task["gitlab:backup:#{subtask}:restore"]).to receive(:invoke)
+          end
           expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
         end
 
@@ -95,7 +91,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
     context 'when the restore directory is not empty' do
       before do
         # We only need a backup of the repositories for this test
-        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
+        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,terraform_state,registry')
 
         create(:project, :repository)
       end
@@ -139,11 +135,10 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive(:invoke)
         expect(Rake::Task['gitlab:backup:pages:restore']).to receive(:invoke)
         expect(Rake::Task['gitlab:backup:lfs:restore']).to receive(:invoke)
+        expect(Rake::Task['gitlab:backup:terraform_state:restore']).to receive(:invoke)
         expect(Rake::Task['gitlab:backup:registry:restore']).to receive(:invoke)
+        expect(Rake::Task['gitlab:backup:packages:restore']).to receive(:invoke)
         expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
-
-        # We only need a backup of the repositories for this test
-        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
       end
 
       it 'restores the data' do
@@ -202,10 +197,8 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       end
 
       context 'specific backup tasks' do
-        let(:task_list) { %w(db repo uploads builds artifacts pages lfs registry) }
-
         it 'prints a progress message to stdout' do
-          task_list.each do |task|
+          backup_types.each do |task|
             expect { run_rake_task("gitlab:backup:#{task}:create") }.to output(/Dumping /).to_stdout_from_any_process
           end
         end
@@ -219,16 +212,49 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping artifacts ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping pages ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping lfs objects ... ")
+          expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping terraform states ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping container registry images ... ")
-          expect(Gitlab::BackupLogger).to receive(:info).with(message: "done").exactly(7).times
+          expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping packages ... ")
+          expect(Gitlab::BackupLogger).to receive(:info).with(message: "done").exactly(9).times
 
-          task_list.each do |task|
+          backup_types.each do |task|
             run_rake_task("gitlab:backup:#{task}:create")
           end
         end
       end
     end
 
+    describe 'backup create fails' do
+      using RSpec::Parameterized::TableSyntax
+
+      file_backup_error = Backup::FileBackupError.new('/tmp', '/tmp/backup/uploads')
+      config = ActiveRecord::Base.configurations.find_db_config(Rails.env).configuration_hash
+      db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
+      db_backup_error = Backup::DatabaseBackupError.new(config, db_file_name)
+
+      where(:backup_class, :rake_task, :error) do
+        Backup::Database     | 'gitlab:backup:db:create'        | db_backup_error
+        Backup::Builds       | 'gitlab:backup:builds:create'    | file_backup_error
+        Backup::Uploads      | 'gitlab:backup:uploads:create'   | file_backup_error
+        Backup::Artifacts    | 'gitlab:backup:artifacts:create' | file_backup_error
+        Backup::Pages        | 'gitlab:backup:pages:create'     | file_backup_error
+        Backup::Lfs          | 'gitlab:backup:lfs:create'       | file_backup_error
+        Backup::Registry     | 'gitlab:backup:registry:create'  | file_backup_error
+      end
+
+      with_them do
+        before do
+          expect_next_instance_of(backup_class) do |instance|
+            expect(instance).to receive(:dump).and_raise(error)
+          end
+        end
+
+        it "raises an error with message" do
+          expect { run_rake_task(rake_task) }.to output(Regexp.new(error.message)).to_stdout_from_any_process
+        end
+      end
+    end
+
     context 'tar creation' do
       context 'archive file permissions' do
         it 'sets correct permissions on the tar file' do
@@ -255,9 +281,11 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
         tar_contents, exit_status = Gitlab::Popen.popen(
-          %W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz registry.tar.gz}
+          %W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz registry.tar.gz packages.tar.gz}
         )
 
+        puts "CONTENT: #{tar_contents}"
+
         expect(exit_status).to eq(0)
         expect(tar_contents).to match('db')
         expect(tar_contents).to match('uploads.tar.gz')
@@ -266,7 +294,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         expect(tar_contents).to match('artifacts.tar.gz')
         expect(tar_contents).to match('pages.tar.gz')
         expect(tar_contents).to match('lfs.tar.gz')
+        expect(tar_contents).to match('terraform_state.tar.gz')
         expect(tar_contents).to match('registry.tar.gz')
+        expect(tar_contents).to match('packages.tar.gz')
         expect(tar_contents).not_to match(%r{^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|pages.tar.gz|artifacts.tar.gz|registry.tar.gz)/$})
       end
 
@@ -274,7 +304,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
         temp_dirs = Dir.glob(
-          File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,pages,lfs,registry}')
+          File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,pages,lfs,terraform_state,registry,packages}')
         )
 
         expect(temp_dirs).to be_empty
@@ -304,7 +334,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
 
       before do
         # We only need a backup of the repositories for this test
-        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
+        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,terraform_state,registry')
         stub_storage_settings( second_storage_name => {
           'gitaly_address' => Gitlab.config.repositories.storages.default.gitaly_address,
           'path' => TestEnv::SECOND_STORAGE_PATH
@@ -378,7 +408,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
     context 'concurrency settings' do
       before do
         # We only need a backup of the repositories for this test
-        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
+        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,terraform_state,registry')
 
         create(:project, :repository)
       end
@@ -407,7 +437,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
             .with(max_concurrency: 5, max_storage_concurrency: 2)
             .and_call_original
         end
-        expect(::Backup::GitalyBackup).to receive(:new).with(anything, parallel: 5, parallel_storage: 2).and_call_original
+        expect(::Backup::GitalyBackup).to receive(:new).with(anything, max_parallelism: 5, storage_parallelism: 2).and_call_original
 
         expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
       end
@@ -425,31 +455,34 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
   end
   # backup_create task
 
-  describe "Skipping items" do
+  describe "Skipping items in a backup" do
     before do
-      stub_env('SKIP', 'repositories,uploads')
+      stub_env('SKIP', 'an-unknown-type,repositories,uploads,anotherunknowntype')
 
       create(:project, :repository)
     end
 
-    it "does not contain skipped item" do
+    it "does not contain repositories and uploads" do
       expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
       tar_contents, _exit_status = Gitlab::Popen.popen(
-        %W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz registry.tar.gz}
+        %W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz registry.tar.gz packages.tar.gz}
       )
 
       expect(tar_contents).to match('db/')
-      expect(tar_contents).to match('uploads.tar.gz')
+      expect(tar_contents).to match('uploads.tar.gz: Not found in archive')
       expect(tar_contents).to match('builds.tar.gz')
       expect(tar_contents).to match('artifacts.tar.gz')
       expect(tar_contents).to match('lfs.tar.gz')
+      expect(tar_contents).to match('terraform_state.tar.gz')
       expect(tar_contents).to match('pages.tar.gz')
       expect(tar_contents).to match('registry.tar.gz')
+      expect(tar_contents).to match('packages.tar.gz')
       expect(tar_contents).not_to match('repositories/')
+      expect(tar_contents).to match('repositories: Not found in archive')
     end
 
-    it 'does not invoke repositories restore' do
+    it 'does not invoke restore of repositories and uploads' do
       expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
       allow(Rake::Task['gitlab:shell:setup'])
@@ -463,7 +496,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive :invoke
       expect(Rake::Task['gitlab:backup:pages:restore']).to receive :invoke
       expect(Rake::Task['gitlab:backup:lfs:restore']).to receive :invoke
+      expect(Rake::Task['gitlab:backup:terraform_state:restore']).to receive :invoke
       expect(Rake::Task['gitlab:backup:registry:restore']).to receive :invoke
+      expect(Rake::Task['gitlab:backup:packages:restore']).to receive :invoke
       expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
       expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
     end
@@ -488,8 +523,10 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         'builds.tar.gz',
         'artifacts.tar.gz',
         'lfs.tar.gz',
+        'terraform_state.tar.gz',
         'pages.tar.gz',
         'registry.tar.gz',
+        'packages.tar.gz',
         'repositories'
       )
     end
@@ -501,14 +538,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         .to receive(:invoke).and_return(true)
 
       expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:db:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:repo:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:uploads:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:builds:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:pages:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:lfs:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:registry:restore']).to receive :invoke
+      backup_types.each do |subtask|
+        expect(Rake::Task["gitlab:backup:#{subtask}:restore"]).to receive :invoke
+      end
       expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
       expect { run_rake_task("gitlab:backup:restore") }.to output.to_stdout_from_any_process
     end
diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb
index 830d0dded2e..92c896b1ab0 100644
--- a/spec/tasks/gitlab/db_rake_spec.rb
+++ b/spec/tasks/gitlab/db_rake_spec.rb
@@ -214,7 +214,7 @@ RSpec.describe 'gitlab:db namespace rake task', :silence_stdout do
         expect(Gitlab::Database::Reindexing).to receive(:enabled?).and_return(false)
         expect(Gitlab::Database::Reindexing).not_to receive(:invoke)
 
-        run_rake_task('gitlab:db:reindex')
+        expect { run_rake_task('gitlab:db:reindex') }.to raise_error(SystemExit)
       end
     end
   end
@@ -233,7 +233,7 @@ RSpec.describe 'gitlab:db namespace rake task', :silence_stdout do
           expect(Gitlab::Database::Reindexing).to receive(:enabled?).and_return(false)
           expect(Gitlab::Database::Reindexing).not_to receive(:invoke).with(database_name)
 
-          run_rake_task("gitlab:db:reindex:#{database_name}")
+          expect { run_rake_task("gitlab:db:reindex:#{database_name}") }.to raise_error(SystemExit)
         end
       end
     end
diff --git a/spec/tasks/gitlab/password_rake_spec.rb b/spec/tasks/gitlab/password_rake_spec.rb
index 65bba836024..ec18d713351 100644
--- a/spec/tasks/gitlab/password_rake_spec.rb
+++ b/spec/tasks/gitlab/password_rake_spec.rb
@@ -3,7 +3,7 @@
 require 'rake_helper'
 
 RSpec.describe 'gitlab:password rake tasks', :silence_stdout do
-  let_it_be(:user_1) { create(:user, username: 'foobar', password: 'initial_password') }
+  let_it_be(:user_1) { create(:user, username: 'foobar', password: Gitlab::Password.test_default) }
 
   def stub_username(username)
     allow(Gitlab::TaskHelpers).to receive(:prompt).with('Enter username: ').and_return(username)
@@ -19,14 +19,14 @@ RSpec.describe 'gitlab:password rake tasks', :silence_stdout do
     Rake.application.rake_require 'tasks/gitlab/password'
 
     stub_username('foobar')
-    stub_password('secretpassword')
+    stub_password(Gitlab::Password.test_default)
   end
 
   describe ':reset' do
     context 'when all inputs are correct' do
       it 'updates the password properly' do
         run_rake_task('gitlab:password:reset', user_1.username)
-        expect(user_1.reload.valid_password?('secretpassword')).to eq(true)
+        expect(user_1.reload.valid_password?(Gitlab::Password.test_default)).to eq(true)
       end
     end
 
@@ -55,7 +55,7 @@ RSpec.describe 'gitlab:password rake tasks', :silence_stdout do
 
     context 'when passwords do not match' do
       before do
-        stub_password('randompassword', 'differentpassword')
+        stub_password(Gitlab::Password.test_default, "different" + Gitlab::Password.test_default)
       end
 
       it 'aborts with an error' do
diff --git a/spec/tasks/gitlab/usage_data_rake_spec.rb b/spec/tasks/gitlab/usage_data_rake_spec.rb
index acaf9b5729b..442b884b313 100644
--- a/spec/tasks/gitlab/usage_data_rake_spec.rb
+++ b/spec/tasks/gitlab/usage_data_rake_spec.rb
@@ -9,6 +9,7 @@ RSpec.describe 'gitlab:usage data take tasks', :silence_stdout do
     Rake.application.rake_require 'tasks/gitlab/usage_data'
     # stub prometheus external http calls https://gitlab.com/gitlab-org/gitlab/-/issues/245277
     stub_prometheus_queries
+    stub_database_flavor_check
   end
 
   describe 'dump_sql_in_yaml' do