Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42

10 files changed, 98 insertions, 76 deletions

diff --git a/spec/lib/backup/artifacts_spec.rb b/spec/lib/backup/artifacts_spec.rb
index 5a965030b01..102d787a5e1 100644
--- a/spec/lib/backup/artifacts_spec.rb
+++ b/spec/lib/backup/artifacts_spec.rb
@@ -12,7 +12,7 @@ RSpec.describe Backup::Artifacts do
       Dir.mktmpdir do |tmpdir|
         allow(JobArtifactUploader).to receive(:root) { "#{tmpdir}" }
 
-        expect(backup.app_files_dir).to eq("#{tmpdir}")
+        expect(backup.app_files_dir).to eq("#{File.realpath(tmpdir)}")
       end
     end
   end
diff --git a/spec/lib/backup/files_spec.rb b/spec/lib/backup/files_spec.rb
index 92de191da2d..6bff0919293 100644
--- a/spec/lib/backup/files_spec.rb
+++ b/spec/lib/backup/files_spec.rb
@@ -134,7 +134,7 @@ RSpec.describe Backup::Files do
 
       expect do
         subject.dump
-      end.to raise_error(/Backup operation failed:/)
+      end.to raise_error(/Failed to create compressed file/)
     end
 
     describe 'with STRATEGY=copy' do
@@ -170,7 +170,7 @@ RSpec.describe Backup::Files do
         expect do
           subject.dump
         end.to output(/rsync failed/).to_stdout
-           .and raise_error(/Backup failed/)
+           .and raise_error(/Failed to create compressed file/)
       end
     end
   end
diff --git a/spec/lib/backup/gitaly_backup_spec.rb b/spec/lib/backup/gitaly_backup_spec.rb
index 2ccde517533..cd0d984fbdb 100644
--- a/spec/lib/backup/gitaly_backup_spec.rb
+++ b/spec/lib/backup/gitaly_backup_spec.rb
@@ -3,8 +3,8 @@
 require 'spec_helper'
 
 RSpec.describe Backup::GitalyBackup do
-  let(:parallel) { nil }
-  let(:parallel_storage) { nil }
+  let(:max_parallelism) { nil }
+  let(:storage_parallelism) { nil }
 
   let(:progress) do
     Tempfile.new('progress').tap do |progress|
@@ -23,7 +23,7 @@ RSpec.describe Backup::GitalyBackup do
     progress.close
   end
 
-  subject { described_class.new(progress, parallel: parallel, parallel_storage: parallel_storage) }
+  subject { described_class.new(progress, max_parallelism: max_parallelism, storage_parallelism: storage_parallelism) }
 
   context 'unknown' do
     it 'fails to start unknown' do
@@ -48,7 +48,7 @@ RSpec.describe Backup::GitalyBackup do
         subject.enqueue(project, Gitlab::GlRepository::DESIGN)
         subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET)
         subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
-        subject.wait
+        subject.finish!
 
         expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle'))
         expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.wiki.bundle'))
@@ -58,24 +58,24 @@ RSpec.describe Backup::GitalyBackup do
       end
 
       context 'parallel option set' do
-        let(:parallel) { 3 }
+        let(:max_parallelism) { 3 }
 
         it 'passes parallel option through' do
           expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel', '3').and_call_original
 
           subject.start(:create)
-          subject.wait
+          subject.finish!
         end
       end
 
       context 'parallel_storage option set' do
-        let(:parallel_storage) { 3 }
+        let(:storage_parallelism) { 3 }
 
         it 'passes parallel option through' do
           expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel-storage', '3').and_call_original
 
           subject.start(:create)
-          subject.wait
+          subject.finish!
         end
       end
 
@@ -83,7 +83,7 @@ RSpec.describe Backup::GitalyBackup do
         expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false'))
 
         subject.start(:create)
-        expect { subject.wait }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
+        expect { subject.finish! }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
       end
     end
 
@@ -115,7 +115,7 @@ RSpec.describe Backup::GitalyBackup do
         expect(Open3).to receive(:popen2).with(ssl_env, anything, 'create', '-path', anything).and_call_original
 
         subject.start(:create)
-        subject.wait
+        subject.finish!
       end
     end
   end
@@ -145,7 +145,7 @@ RSpec.describe Backup::GitalyBackup do
       subject.enqueue(project, Gitlab::GlRepository::DESIGN)
       subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET)
       subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
-      subject.wait
+      subject.finish!
 
       collect_commit_shas = -> (repo) { repo.commits('master', limit: 10).map(&:sha) }
 
@@ -157,24 +157,24 @@ RSpec.describe Backup::GitalyBackup do
     end
 
     context 'parallel option set' do
-      let(:parallel) { 3 }
+      let(:max_parallelism) { 3 }
 
       it 'passes parallel option through' do
         expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel', '3').and_call_original
 
         subject.start(:restore)
-        subject.wait
+        subject.finish!
       end
     end
 
     context 'parallel_storage option set' do
-      let(:parallel_storage) { 3 }
+      let(:storage_parallelism) { 3 }
 
       it 'passes parallel option through' do
         expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel-storage', '3').and_call_original
 
         subject.start(:restore)
-        subject.wait
+        subject.finish!
       end
     end
 
@@ -182,7 +182,7 @@ RSpec.describe Backup::GitalyBackup do
       expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false'))
 
       subject.start(:restore)
-      expect { subject.wait }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
+      expect { subject.finish! }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
     end
   end
 end
diff --git a/spec/lib/backup/gitaly_rpc_backup_spec.rb b/spec/lib/backup/gitaly_rpc_backup_spec.rb
index fb442f4a86f..14f9d27ca6e 100644
--- a/spec/lib/backup/gitaly_rpc_backup_spec.rb
+++ b/spec/lib/backup/gitaly_rpc_backup_spec.rb
@@ -33,7 +33,7 @@ RSpec.describe Backup::GitalyRpcBackup do
         subject.enqueue(project, Gitlab::GlRepository::DESIGN)
         subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET)
         subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
-        subject.wait
+        subject.finish!
 
         expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle'))
         expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.wiki.bundle'))
@@ -52,7 +52,7 @@ RSpec.describe Backup::GitalyRpcBackup do
         it 'logs an appropriate message', :aggregate_failures do
           subject.start(:create)
           subject.enqueue(project, Gitlab::GlRepository::PROJECT)
-          subject.wait
+          subject.finish!
 
           expect(progress).to have_received(:puts).with("[Failed] backing up #{project.full_path} (#{project.disk_path})")
           expect(progress).to have_received(:puts).with("Error Fail in tests")
@@ -96,7 +96,7 @@ RSpec.describe Backup::GitalyRpcBackup do
       subject.enqueue(project, Gitlab::GlRepository::DESIGN)
       subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET)
       subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
-      subject.wait
+      subject.finish!
 
       collect_commit_shas = -> (repo) { repo.commits('master', limit: 10).map(&:sha) }
 
@@ -129,7 +129,7 @@ RSpec.describe Backup::GitalyRpcBackup do
       subject.enqueue(project, Gitlab::GlRepository::DESIGN)
       subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET)
       subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
-      subject.wait
+      subject.finish!
     end
 
     context 'failure' do
@@ -143,7 +143,7 @@ RSpec.describe Backup::GitalyRpcBackup do
       it 'logs an appropriate message', :aggregate_failures do
         subject.start(:restore)
         subject.enqueue(project, Gitlab::GlRepository::PROJECT)
-        subject.wait
+        subject.finish!
 
         expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} (#{project.disk_path})")
         expect(progress).to have_received(:puts).with("Error Fail in tests")
diff --git a/spec/lib/backup/lfs_spec.rb b/spec/lib/backup/lfs_spec.rb
new file mode 100644
index 00000000000..fdc1c0c885d
--- /dev/null
+++ b/spec/lib/backup/lfs_spec.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Backup::Lfs do
+  let(:progress) { StringIO.new }
+
+  subject(:backup) { described_class.new(progress) }
+
+  describe '#dump' do
+    before do
+      allow(File).to receive(:realpath).and_call_original
+      allow(File).to receive(:realpath).with('/var/lfs-objects').and_return('/var/lfs-objects')
+      allow(File).to receive(:realpath).with('/var/lfs-objects/..').and_return('/var')
+      allow(Settings.lfs).to receive(:storage_path).and_return('/var/lfs-objects')
+    end
+
+    it 'uses the correct lfs dir in tar command', :aggregate_failures do
+      expect(backup.app_files_dir).to eq('/var/lfs-objects')
+      expect(backup).to receive(:tar).and_return('blabla-tar')
+      expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found -C /var/lfs-objects -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
+      expect(backup).to receive(:pipeline_succeeded?).and_return(true)
+
+      backup.dump
+    end
+  end
+end
diff --git a/spec/lib/backup/manager_spec.rb b/spec/lib/backup/manager_spec.rb
index 32eea82cfdf..31cc3012eb1 100644
--- a/spec/lib/backup/manager_spec.rb
+++ b/spec/lib/backup/manager_spec.rb
@@ -15,7 +15,7 @@ RSpec.describe Backup::Manager do
   end
 
   describe '#pack' do
-    let(:expected_backup_contents) { %w(repositories db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml) }
+    let(:expected_backup_contents) { %w(repositories db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz packages.tar.gz backup_information.yml) }
     let(:tar_file) { '1546300800_2019_01_01_12.3_gitlab_backup.tar' }
     let(:tar_system_options) { { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] } }
     let(:tar_cmdline) { ['tar', '-cf', '-', *expected_backup_contents, tar_system_options] }
@@ -57,7 +57,7 @@ RSpec.describe Backup::Manager do
     end
 
     context 'when skipped is set in backup_information.yml' do
-      let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml} }
+      let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz packages.tar.gz backup_information.yml} }
       let(:backup_information) do
         {
           backup_created_at: Time.zone.parse('2019-01-01'),
@@ -74,7 +74,7 @@ RSpec.describe Backup::Manager do
     end
 
     context 'when a directory does not exist' do
-      let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml} }
+      let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz packages.tar.gz backup_information.yml} }
 
       before do
         expect(Dir).to receive(:exist?).with(File.join(Gitlab.config.backup.path, 'repositories')).and_return(false)
diff --git a/spec/lib/backup/object_backup_spec.rb b/spec/lib/backup/object_backup_spec.rb
new file mode 100644
index 00000000000..6192b5c3482
--- /dev/null
+++ b/spec/lib/backup/object_backup_spec.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.shared_examples 'backup object' do |setting|
+  let(:progress) { StringIO.new }
+  let(:backup_path) { "/var/#{setting}" }
+
+  subject(:backup) { described_class.new(progress) }
+
+  describe '#dump' do
+    before do
+      allow(File).to receive(:realpath).and_call_original
+      allow(File).to receive(:realpath).with(backup_path).and_return(backup_path)
+      allow(File).to receive(:realpath).with("#{backup_path}/..").and_return('/var')
+      allow(Settings.send(setting)).to receive(:storage_path).and_return(backup_path)
+    end
+
+    it 'uses the correct storage dir in tar command and excludes tmp', :aggregate_failures do
+      expect(backup.app_files_dir).to eq(backup_path)
+      expect(backup).to receive(:tar).and_return('blabla-tar')
+      expect(backup).to receive(:run_pipeline!).with([%W(blabla-tar --exclude=lost+found --exclude=./tmp -C #{backup_path} -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
+      expect(backup).to receive(:pipeline_succeeded?).and_return(true)
+
+      backup.dump
+    end
+  end
+end
+
+RSpec.describe Backup::Packages do
+  it_behaves_like 'backup object', 'packages'
+end
+
+RSpec.describe Backup::TerraformState do
+  it_behaves_like 'backup object', 'terraform_state'
+end
diff --git a/spec/lib/backup/repositories_spec.rb b/spec/lib/backup/repositories_spec.rb
index 85818038c9d..f3830da344b 100644
--- a/spec/lib/backup/repositories_spec.rb
+++ b/spec/lib/backup/repositories_spec.rb
@@ -25,7 +25,7 @@ RSpec.describe Backup::Repositories do
         expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::DESIGN)
         expect(strategy).to have_received(:enqueue).with(project_snippet, Gitlab::GlRepository::SNIPPET)
         expect(strategy).to have_received(:enqueue).with(personal_snippet, Gitlab::GlRepository::SNIPPET)
-        expect(strategy).to have_received(:wait)
+        expect(strategy).to have_received(:finish!)
       end
     end
 
@@ -49,7 +49,7 @@ RSpec.describe Backup::Repositories do
         projects.each do |project|
           expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
         end
-        expect(strategy).to receive(:wait)
+        expect(strategy).to receive(:finish!)
 
         subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
       end
@@ -91,7 +91,7 @@ RSpec.describe Backup::Repositories do
         projects.each do |project|
           expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
         end
-        expect(strategy).to receive(:wait)
+        expect(strategy).to receive(:finish!)
 
         subject.dump(max_concurrency: 2, max_storage_concurrency: 2)
       end
@@ -114,7 +114,7 @@ RSpec.describe Backup::Repositories do
           projects.each do |project|
             expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
           end
-          expect(strategy).to receive(:wait)
+          expect(strategy).to receive(:finish!)
 
           subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency)
         end
@@ -128,7 +128,7 @@ RSpec.describe Backup::Repositories do
           projects.each do |project|
             expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
           end
-          expect(strategy).to receive(:wait)
+          expect(strategy).to receive(:finish!)
 
           subject.dump(max_concurrency: 3, max_storage_concurrency: max_storage_concurrency)
         end
@@ -184,7 +184,7 @@ RSpec.describe Backup::Repositories do
       expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::DESIGN)
       expect(strategy).to have_received(:enqueue).with(project_snippet, Gitlab::GlRepository::SNIPPET)
       expect(strategy).to have_received(:enqueue).with(personal_snippet, Gitlab::GlRepository::SNIPPET)
-      expect(strategy).to have_received(:wait)
+      expect(strategy).to have_received(:finish!)
     end
 
     context 'restoring object pools' do
diff --git a/spec/lib/backup/repository_backup_error_spec.rb b/spec/lib/backup/repository_backup_error_spec.rb
deleted file mode 100644
index 44c75c1cf77..00000000000
--- a/spec/lib/backup/repository_backup_error_spec.rb
+++ /dev/null
@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Backup::RepositoryBackupError do
-  let_it_be(:snippet) { create(:snippet, content: 'foo', file_name: 'foo') }
-  let_it_be(:project) { create(:project, :repository) }
-  let_it_be(:wiki) { ProjectWiki.new(project, nil ) }
-
-  let(:backup_repos_path) { '/tmp/backup/repositories' }
-
-  shared_examples 'includes backup path' do
-    it { is_expected.to respond_to :container }
-    it { is_expected.to respond_to :backup_repos_path }
-
-    it 'expects exception message to include repo backup path location' do
-      expect(subject.message).to include("#{subject.backup_repos_path}")
-    end
-
-    it 'expects exception message to include container being back-up' do
-      expect(subject.message).to include("#{subject.container.disk_path}")
-    end
-  end
-
-  context 'with snippet repository' do
-    subject { described_class.new(snippet, backup_repos_path) }
-
-    it_behaves_like 'includes backup path'
-  end
-
-  context 'with project repository' do
-    subject { described_class.new(project, backup_repos_path) }
-
-    it_behaves_like 'includes backup path'
-  end
-
-  context 'with wiki repository' do
-    subject { described_class.new(wiki, backup_repos_path) }
-
-    it_behaves_like 'includes backup path'
-  end
-end
diff --git a/spec/lib/backup/uploads_spec.rb b/spec/lib/backup/uploads_spec.rb
index a82cb764f4d..c173916fe91 100644
--- a/spec/lib/backup/uploads_spec.rb
+++ b/spec/lib/backup/uploads_spec.rb
@@ -14,13 +14,14 @@ RSpec.describe Backup::Uploads do
 
         allow(Gitlab.config.uploads).to receive(:storage_path) { tmpdir }
 
-        expect(backup.app_files_dir).to eq("#{tmpdir}/uploads")
+        expect(backup.app_files_dir).to eq("#{File.realpath(tmpdir)}/uploads")
       end
     end
   end
 
   describe '#dump' do
     before do
+      allow(File).to receive(:realpath).and_call_original
       allow(File).to receive(:realpath).with('/var/uploads').and_return('/var/uploads')
       allow(File).to receive(:realpath).with('/var/uploads/..').and_return('/var')
       allow(Gitlab.config.uploads).to receive(:storage_path) { '/var' }