Add latest changes from gitlab-org/gitlab@14-6-stable-eev14.6.0-rc42

5 files changed, 122 insertions, 52 deletions

diff --git a/spec/tasks/gitlab/background_migrations_rake_spec.rb b/spec/tasks/gitlab/background_migrations_rake_spec.rb
new file mode 100644
index 00000000000..079b4d3aea8
--- /dev/null
+++ b/spec/tasks/gitlab/background_migrations_rake_spec.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'rake_helper'
+
+RSpec.describe 'gitlab:background_migrations namespace rake tasks' do
+  before do
+    Rake.application.rake_require 'tasks/gitlab/background_migrations'
+  end
+
+  describe 'finalize' do
+    subject(:finalize_task) { run_rake_task('gitlab:background_migrations:finalize', *arguments) }
+
+    context 'without the proper arguments' do
+      let(:arguments) { %w[CopyColumnUsingBackgroundMigrationJob events id] }
+
+      it 'exits without finalizing the migration' do
+        expect(Gitlab::Database::BackgroundMigration::BatchedMigrationRunner).not_to receive(:finalize)
+
+        expect { finalize_task }.to output(/Must specify job_arguments as an argument/).to_stdout
+          .and raise_error(SystemExit) { |error| expect(error.status).to eq(1) }
+      end
+    end
+
+    context 'with the proper arguments' do
+      let(:arguments) { %w[CopyColumnUsingBackgroundMigrationJob events id [["id1"\,"id2"]]] }
+
+      it 'finalizes the matching migration' do
+        expect(Gitlab::Database::BackgroundMigration::BatchedMigrationRunner).to receive(:finalize)
+          .with('CopyColumnUsingBackgroundMigrationJob', 'events', 'id', [%w[id1 id2]])
+
+        expect { finalize_task }.to output(/Done/).to_stdout
+      end
+    end
+  end
+
+  describe 'status' do
+    subject(:status_task) { run_rake_task('gitlab:background_migrations:status') }
+
+    it 'outputs the status of background migrations' do
+      migration1 = create(:batched_background_migration, :finished, job_arguments: [%w[id1 id2]])
+      migration2 = create(:batched_background_migration, :failed, job_arguments: [])
+
+      expect { status_task }.to output(<<~OUTPUT).to_stdout
+        finished   | #{migration1.job_class_name},#{migration1.table_name},#{migration1.column_name},[["id1","id2"]]
+        failed     | #{migration2.job_class_name},#{migration2.table_name},#{migration2.column_name},[]
+      OUTPUT
+    end
+  end
+end
diff --git a/spec/tasks/gitlab/cleanup_rake_spec.rb b/spec/tasks/gitlab/cleanup_rake_spec.rb
index 16c907ca87c..bd4d9643433 100644
--- a/spec/tasks/gitlab/cleanup_rake_spec.rb
+++ b/spec/tasks/gitlab/cleanup_rake_spec.rb
@@ -166,17 +166,17 @@ RSpec.describe 'gitlab:cleanup rake tasks', :silence_stdout do
   end
 
   context 'sessions' do
-    describe 'gitlab:cleanup:sessions:active_sessions_lookup_keys', :clean_gitlab_redis_shared_state do
+    describe 'gitlab:cleanup:sessions:active_sessions_lookup_keys', :clean_gitlab_redis_sessions do
       subject(:rake_task) { run_rake_task('gitlab:cleanup:sessions:active_sessions_lookup_keys') }
 
       let!(:user) { create(:user) }
       let(:existing_session_id) { '5' }
 
       before do
-        Gitlab::Redis::SharedState.with do |redis|
-          redis.set("session:user:gitlab:#{user.id}:#{existing_session_id}",
-                    Marshal.dump(true))
-          redis.sadd("session:lookup:user:gitlab:#{user.id}", (1..10).to_a)
+        Gitlab::Redis::Sessions.with do |redis|
+          redis.set(ActiveSession.key_name(user.id, existing_session_id),
+                    ActiveSession.new(session_id: 'x').dump)
+          redis.sadd(ActiveSession.lookup_key_name(user.id), (1..10).to_a)
         end
       end
 
@@ -185,11 +185,11 @@ RSpec.describe 'gitlab:cleanup rake tasks', :silence_stdout do
       end
 
       it 'removes expired active session lookup keys' do
-        Gitlab::Redis::SharedState.with do |redis|
-          lookup_key = "session:lookup:user:gitlab:#{user.id}"
+        Gitlab::Redis::Sessions.with do |redis|
+          lookup_key = ActiveSession.lookup_key_name(user.id)
+
           expect { subject }.to change { redis.scard(lookup_key) }.from(10).to(1)
-          expect(redis.smembers("session:lookup:user:gitlab:#{user.id}")).to(
-            eql([existing_session_id]))
+          expect(redis.smembers(lookup_key)).to contain_exactly existing_session_id
         end
       end
     end
diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb
index 38392f77307..830d0dded2e 100644
--- a/spec/tasks/gitlab/db_rake_spec.rb
+++ b/spec/tasks/gitlab/db_rake_spec.rb
@@ -138,6 +138,10 @@ RSpec.describe 'gitlab:db namespace rake task', :silence_stdout do
         stub_file_read(structure_file, content: input)
         allow(File).to receive(:open).with(structure_file.to_s, any_args).and_yield(output)
       end
+
+      if Gitlab.ee?
+        allow(File).to receive(:open).with(Rails.root.join(Gitlab::Database::GEO_DATABASE_DIR, 'structure.sql').to_s, any_args).and_yield(output)
+      end
     end
 
     after do
@@ -191,7 +195,7 @@ RSpec.describe 'gitlab:db namespace rake task', :silence_stdout do
 
     it 'drops extra schemas' do
       Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
-        expect(connection).to receive(:execute).with("DROP SCHEMA IF EXISTS \"#{schema}\"")
+        expect(connection).to receive(:execute).with("DROP SCHEMA IF EXISTS \"#{schema}\" CASCADE")
       end
 
       subject
@@ -199,43 +203,38 @@ RSpec.describe 'gitlab:db namespace rake task', :silence_stdout do
   end
 
   describe 'reindex' do
-    let(:reindex) { double('reindex') }
-    let(:indexes) { double('indexes') }
-    let(:databases) { Gitlab::Database.database_base_models }
-    let(:databases_count) { databases.count }
-
-    it 'cleans up any leftover indexes' do
-      expect(Gitlab::Database::Reindexing).to receive(:cleanup_leftovers!).exactly(databases_count).times
+    it 'delegates to Gitlab::Database::Reindexing' do
+      expect(Gitlab::Database::Reindexing).to receive(:invoke)
 
       run_rake_task('gitlab:db:reindex')
     end
 
-    context 'when async index creation is enabled' do
-      it 'executes async index creation prior to any reindexing actions' do
-        stub_feature_flags(database_async_index_creation: true)
-
-        expect(Gitlab::Database::AsyncIndexes).to receive(:create_pending_indexes!).ordered.exactly(databases_count).times
-        expect(Gitlab::Database::Reindexing).to receive(:automatic_reindexing).ordered.exactly(databases_count).times
+    context 'when reindexing is not enabled' do
+      it 'is a no-op' do
+        expect(Gitlab::Database::Reindexing).to receive(:enabled?).and_return(false)
+        expect(Gitlab::Database::Reindexing).not_to receive(:invoke)
 
         run_rake_task('gitlab:db:reindex')
       end
     end
+  end
 
-    context 'when async index creation is disabled' do
-      it 'does not execute async index creation' do
-        stub_feature_flags(database_async_index_creation: false)
-
-        expect(Gitlab::Database::AsyncIndexes).not_to receive(:create_pending_indexes!)
+  databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml
+  ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database_name|
+    describe "reindex:#{database_name}" do
+      it 'delegates to Gitlab::Database::Reindexing' do
+        expect(Gitlab::Database::Reindexing).to receive(:invoke).with(database_name)
 
-        run_rake_task('gitlab:db:reindex')
+        run_rake_task("gitlab:db:reindex:#{database_name}")
       end
-    end
 
-    context 'calls automatic reindexing' do
-      it 'uses all candidate indexes' do
-        expect(Gitlab::Database::Reindexing).to receive(:automatic_reindexing).exactly(databases_count).times
+      context 'when reindexing is not enabled' do
+        it 'is a no-op' do
+          expect(Gitlab::Database::Reindexing).to receive(:enabled?).and_return(false)
+          expect(Gitlab::Database::Reindexing).not_to receive(:invoke).with(database_name)
 
-        run_rake_task('gitlab:db:reindex')
+          run_rake_task("gitlab:db:reindex:#{database_name}")
+        end
       end
     end
   end
@@ -328,6 +327,32 @@ RSpec.describe 'gitlab:db namespace rake task', :silence_stdout do
     end
   end
 
+  context 'with multiple databases', :reestablished_active_record_base do
+    before do
+      allow(ActiveRecord::Tasks::DatabaseTasks).to receive(:setup_initial_database_yaml).and_return([:main, :geo])
+    end
+
+    describe 'db:structure:dump' do
+      it 'invokes gitlab:db:clean_structure_sql' do
+        skip unless Gitlab.ee?
+
+        expect(Rake::Task['gitlab:db:clean_structure_sql']).to receive(:invoke).twice.and_return(true)
+
+        expect { run_rake_task('db:structure:dump:main') }.not_to raise_error
+      end
+    end
+
+    describe 'db:schema:dump' do
+      it 'invokes gitlab:db:clean_structure_sql' do
+        skip unless Gitlab.ee?
+
+        expect(Rake::Task['gitlab:db:clean_structure_sql']).to receive(:invoke).once.and_return(true)
+
+        expect { run_rake_task('db:schema:dump:main') }.not_to raise_error
+      end
+    end
+  end
+
   def run_rake_task(task_name, arguments = '')
     Rake::Task[task_name].reenable
     Rake.application.invoke_task("#{task_name}#{arguments}")
diff --git a/spec/tasks/gitlab/gitaly_rake_spec.rb b/spec/tasks/gitlab/gitaly_rake_spec.rb
index c5625db922d..70c7ddb1d6e 100644
--- a/spec/tasks/gitlab/gitaly_rake_spec.rb
+++ b/spec/tasks/gitlab/gitaly_rake_spec.rb
@@ -7,26 +7,26 @@ RSpec.describe 'gitlab:gitaly namespace rake task', :silence_stdout do
     Rake.application.rake_require 'tasks/gitlab/gitaly'
   end
 
-  describe 'install' do
-    let(:repo) { 'https://gitlab.com/gitlab-org/gitaly.git' }
-    let(:clone_path) { Rails.root.join('tmp/tests/gitaly').to_s }
-    let(:storage_path) { Rails.root.join('tmp/tests/repositories').to_s }
-    let(:version) { File.read(Rails.root.join(Gitlab::GitalyClient::SERVER_VERSION_FILE)).chomp }
+  let(:repo) { 'https://gitlab.com/gitlab-org/gitaly.git' }
+  let(:clone_path) { Rails.root.join('tmp/tests/gitaly').to_s }
+  let(:storage_path) { Rails.root.join('tmp/tests/repositories').to_s }
+  let(:version) { File.read(Rails.root.join(Gitlab::GitalyClient::SERVER_VERSION_FILE)).chomp }
 
-    subject { run_rake_task('gitlab:gitaly:install', clone_path, storage_path) }
+  describe 'clone' do
+    subject { run_rake_task('gitlab:gitaly:clone', clone_path, storage_path) }
 
     context 'no dir given' do
       it 'aborts and display a help message' do
         # avoid writing task output to spec progress
         allow($stderr).to receive :write
-        expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/
+        expect { run_rake_task('gitlab:gitaly:clone') }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/
       end
     end
 
     context 'no storage path given' do
       it 'aborts and display a help message' do
         allow($stderr).to receive :write
-        expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/
+        expect { run_rake_task('gitlab:gitaly:clone', clone_path) }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/
       end
     end
 
@@ -40,11 +40,6 @@ RSpec.describe 'gitlab:gitaly namespace rake task', :silence_stdout do
     end
 
     describe 'checkout or clone' do
-      before do
-        stub_env('CI', false)
-        expect(Dir).to receive(:chdir).with(clone_path)
-      end
-
       it 'calls checkout_or_clone_version with the right arguments' do
         expect(main_object)
           .to receive(:checkout_or_clone_version).with(version: version, repo: repo, target_dir: clone_path, clone_opts: %w[--depth 1])
@@ -52,6 +47,10 @@ RSpec.describe 'gitlab:gitaly namespace rake task', :silence_stdout do
         subject
       end
     end
+  end
+
+  describe 'install' do
+    subject { run_rake_task('gitlab:gitaly:install', clone_path, storage_path) }
 
     describe 'gmake/make' do
       before do
@@ -62,10 +61,6 @@ RSpec.describe 'gitlab:gitaly namespace rake task', :silence_stdout do
       end
 
       context 'gmake is available' do
-        before do
-          expect(main_object).to receive(:checkout_or_clone_version)
-        end
-
         it 'calls gmake in the gitaly directory' do
           expect(Gitlab::Popen).to receive(:popen)
             .with(%w[which gmake])
@@ -93,7 +88,6 @@ RSpec.describe 'gitlab:gitaly namespace rake task', :silence_stdout do
 
       context 'gmake is not available' do
         before do
-          expect(main_object).to receive(:checkout_or_clone_version)
           expect(Gitlab::Popen).to receive(:popen)
             .with(%w[which gmake])
             .and_return(['', 42])
diff --git a/spec/tasks/gitlab/task_helpers_spec.rb b/spec/tasks/gitlab/task_helpers_spec.rb
index 2921913319b..0c43dd15e8c 100644
--- a/spec/tasks/gitlab/task_helpers_spec.rb
+++ b/spec/tasks/gitlab/task_helpers_spec.rb
@@ -72,6 +72,8 @@ RSpec.describe Gitlab::TaskHelpers do
   describe '#checkout_version' do
     it 'clones the repo in the target dir' do
       expect(subject)
+        .to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} -C #{clone_path} config protocol.version 2])
+      expect(subject)
         .to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} -C #{clone_path} fetch --quiet origin #{tag}])
       expect(subject)
         .to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} -C #{clone_path} checkout -f --quiet FETCH_HEAD --])