Add latest changes from gitlab-org/gitlab@15-5-stable-ee

17 files changed, 113 insertions, 35 deletions

diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 45307c3454b..2244e415c3d 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -814,7 +814,7 @@ end
 #
 Settings['sidekiq'] ||= Settingslogic.new({})
 Settings['sidekiq']['log_format'] ||= 'default'
-Settings['sidekiq']['routing_rules'] = Settings.__send__(:build_sidekiq_routing_rules, Settings['sidekiq']['routing_rules'])
+Settings['sidekiq']['routing_rules'] ||= []
 
 #
 # GitLab Shell
diff --git a/config/settings.rb b/config/settings.rb
index b242e970cc6..51d54817646 100644
--- a/config/settings.rb
+++ b/config/settings.rb
@@ -204,11 +204,5 @@ class Settings < Settingslogic
 
       "#{minute} #{hour} * * #{day_of_week}"
     end
-
-    # Route all jobs to 'default' queue. This setting is meant for self-managed instances use to keep things simple.
-    # See https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1491
-    def build_sidekiq_routing_rules(rules)
-      rules.nil? || rules&.empty? ? [['*', 'default']] : rules
-    end
   end
 end
diff --git a/spec/config/settings_spec.rb b/spec/config/settings_spec.rb
index 9b721d8cfca..fe2081fa5de 100644
--- a/spec/config/settings_spec.rb
+++ b/spec/config/settings_spec.rb
@@ -151,20 +151,6 @@ RSpec.describe Settings do
     end
   end
 
-  describe '.build_sidekiq_routing_rules' do
-    [
-      [nil, [['*', 'default']]],
-      [[], [['*', 'default']]],
-      [[['name=foobar', 'foobar']], [['name=foobar', 'foobar']]]
-    ].each do |input_rules, output_rules|
-      context "Given input routing_rules #{input_rules}" do
-        it "returns output routing_rules #{output_rules}" do
-          expect(described_class.send(:build_sidekiq_routing_rules, input_rules)).to eq(output_rules)
-        end
-      end
-    end
-  end
-
   describe '.microsoft_graph_mailer' do
     it 'defaults' do
       expect(described_class.microsoft_graph_mailer.enabled).to be false
diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb
index cc730e203f6..09548d21106 100644
--- a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb
+++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb
@@ -41,10 +41,10 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::Server, :clean_gitlab_r
     describe '#call' do
       it 'removes the stored job from redis before execution' do
         bare_job = { 'class' => 'TestDeduplicationWorker', 'args' => ['hello'] }
-        job_definition = Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(bare_job.dup, 'default')
+        job_definition = Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(bare_job.dup, 'test_deduplication')
 
         expect(Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob)
-          .to receive(:new).with(a_hash_including(bare_job), 'default')
+          .to receive(:new).with(a_hash_including(bare_job), 'test_deduplication')
                 .and_return(job_definition).twice # once in client middleware
 
         expect(job_definition).to receive(:delete!).ordered.and_call_original
@@ -60,10 +60,10 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::Server, :clean_gitlab_r
 
     it 'removes the stored job from redis after execution' do
       bare_job = { 'class' => 'TestDeduplicationWorker', 'args' => ['hello'] }
-      job_definition = Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(bare_job.dup, 'default')
+      job_definition = Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(bare_job.dup, 'test_deduplication')
 
       expect(Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob)
-        .to receive(:new).with(a_hash_including(bare_job), 'default')
+        .to receive(:new).with(a_hash_including(bare_job), 'test_deduplication')
               .and_return(job_definition).twice # once in client middleware
 
       expect(TestDeduplicationWorker).to receive(:work).ordered.and_call_original
diff --git a/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb b/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb
index a576cf3e2ab..d4391d3023a 100644
--- a/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb
+++ b/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb
@@ -46,7 +46,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do
           expect(migrator.execute('PostReceive' => 'new_queue')).to eq(scanned: 3, migrated: 0)
 
           expect(set_after.length).to eq(3)
-          expect(set_after.map(&:first)).to all(include('queue' => 'default',
+          expect(set_after.map(&:first)).to all(include('queue' => 'authorized_projects',
                                                         'class' => 'AuthorizedProjectsWorker'))
         end
       end
@@ -62,7 +62,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do
               if item['class'] == 'AuthorizedProjectsWorker'
                 expect(item).to include('queue' => 'new_queue', 'args' => [i])
               else
-                expect(item).to include('queue' => 'default', 'args' => [i])
+                expect(item).to include('queue' => 'post_receive', 'args' => [i])
               end
 
               expect(score).to be_within(schedule_jitter).of(i.succ.hours.from_now.to_i)
@@ -116,7 +116,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do
             expect(migrator.execute('PostReceive' => 'new_queue')).to eq(scanned: 4, migrated: 0)
 
             expect(set_after.length).to eq(3)
-            expect(set_after.map(&:first)).to all(include('queue' => 'default'))
+            expect(set_after.map(&:first)).to all(include('queue' => 'authorized_projects'))
           end
         end
 
@@ -138,7 +138,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do
           expect(migrator.execute('PostReceive' => 'new_queue')).to eq(scanned: 4, migrated: 1)
 
           expect(set_after.group_by { |job| job.first['queue'] }.transform_values(&:count))
-            .to eq('default' => 6, 'new_queue' => 1)
+            .to eq('authorized_projects' => 6, 'new_queue' => 1)
         end
 
         it 'iterates through the entire set of jobs' do
diff --git a/spec/lib/gitlab/sidekiq_queue_spec.rb b/spec/lib/gitlab/sidekiq_queue_spec.rb
index 93632848788..5e91282612e 100644
--- a/spec/lib/gitlab/sidekiq_queue_spec.rb
+++ b/spec/lib/gitlab/sidekiq_queue_spec.rb
@@ -4,15 +4,15 @@ require 'spec_helper'
 
 RSpec.describe Gitlab::SidekiqQueue, :clean_gitlab_redis_queues do
   around do |example|
-    Sidekiq::Queue.new('foobar').clear
+    Sidekiq::Queue.new('default').clear
     Sidekiq::Testing.disable!(&example)
-    Sidekiq::Queue.new('foobar').clear
+    Sidekiq::Queue.new('default').clear
   end
 
   def add_job(args, user:, klass: 'AuthorizedProjectsWorker')
     Sidekiq::Client.push(
       'class' => klass,
-      'queue' => 'foobar',
+      'queue' => 'default',
       'args' => args,
       'meta.user' => user.username
     )
@@ -20,7 +20,7 @@ RSpec.describe Gitlab::SidekiqQueue, :clean_gitlab_redis_queues do
 
   describe '#drop_jobs!' do
     shared_examples 'queue processing' do
-      let(:sidekiq_queue) { described_class.new('foobar') }
+      let(:sidekiq_queue) { described_class.new('default') }
       let_it_be(:sidekiq_queue_user) { create(:user) }
 
       before do
@@ -80,7 +80,7 @@ RSpec.describe Gitlab::SidekiqQueue, :clean_gitlab_redis_queues do
       it 'raises NoMetadataError' do
         add_job([1], user: create(:user))
 
-        expect { described_class.new('foobar').drop_jobs!({ username: 'sidekiq_queue_user' }, timeout: 1) }
+        expect { described_class.new('default').drop_jobs!({ username: 'sidekiq_queue_user' }, timeout: 1) }
           .to raise_error(described_class::NoMetadataError)
       end
     end
diff --git a/spec/support/rspec_order_todo.yml b/spec/support/rspec_order_todo.yml
index c4377e368ee..30220b04fa2 100644
--- a/spec/support/rspec_order_todo.yml
+++ b/spec/support/rspec_order_todo.yml
@@ -3442,6 +3442,7 @@
 - './ee/spec/workers/concerns/elastic/indexing_control_spec.rb'
 - './ee/spec/workers/concerns/elastic/migration_obsolete_spec.rb'
 - './ee/spec/workers/concerns/elastic/migration_options_spec.rb'
+- './ee/spec/workers/concerns/geo_queue_spec.rb'
 - './ee/spec/workers/concerns/update_orchestration_policy_configuration_spec.rb'
 - './ee/spec/workers/create_github_webhook_worker_spec.rb'
 - './ee/spec/workers/deployments/auto_rollback_worker_spec.rb'
@@ -10862,14 +10863,18 @@
 - './spec/workers/cluster_wait_for_ingress_ip_address_worker_spec.rb'
 - './spec/workers/concerns/application_worker_spec.rb'
 - './spec/workers/concerns/cluster_agent_queue_spec.rb'
+- './spec/workers/concerns/cluster_queue_spec.rb'
 - './spec/workers/concerns/cronjob_queue_spec.rb'
 - './spec/workers/concerns/gitlab/github_import/object_importer_spec.rb'
+- './spec/workers/concerns/gitlab/github_import/queue_spec.rb'
 - './spec/workers/concerns/gitlab/github_import/rescheduling_methods_spec.rb'
 - './spec/workers/concerns/gitlab/github_import/stage_methods_spec.rb'
 - './spec/workers/concerns/gitlab/notify_upon_death_spec.rb'
 - './spec/workers/concerns/limited_capacity/job_tracker_spec.rb'
 - './spec/workers/concerns/limited_capacity/worker_spec.rb'
 - './spec/workers/concerns/packages/cleanup_artifact_worker_spec.rb'
+- './spec/workers/concerns/pipeline_background_queue_spec.rb'
+- './spec/workers/concerns/pipeline_queue_spec.rb'
 - './spec/workers/concerns/project_import_options_spec.rb'
 - './spec/workers/concerns/reenqueuer_spec.rb'
 - './spec/workers/concerns/repository_check_queue_spec.rb'
diff --git a/spec/workers/concerns/cluster_agent_queue_spec.rb b/spec/workers/concerns/cluster_agent_queue_spec.rb
index 4f67102a0be..b5189cbd8c8 100644
--- a/spec/workers/concerns/cluster_agent_queue_spec.rb
+++ b/spec/workers/concerns/cluster_agent_queue_spec.rb
@@ -14,5 +14,6 @@ RSpec.describe ClusterAgentQueue do
     end
   end
 
+  it { expect(worker.queue).to eq('cluster_agent:example') }
   it { expect(worker.get_feature_category).to eq(:kubernetes_management) }
 end
diff --git a/spec/workers/concerns/cluster_queue_spec.rb b/spec/workers/concerns/cluster_queue_spec.rb
new file mode 100644
index 00000000000..c03ca9cea48
--- /dev/null
+++ b/spec/workers/concerns/cluster_queue_spec.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ClusterQueue do
+  let(:worker) do
+    Class.new do
+      def self.name
+        'DummyWorker'
+      end
+
+      include ApplicationWorker
+      include ClusterQueue
+    end
+  end
+
+  it 'sets a default pipelines queue automatically' do
+    expect(worker.sidekiq_options['queue'])
+      .to eq 'gcp_cluster:dummy'
+  end
+end
diff --git a/spec/workers/concerns/cronjob_queue_spec.rb b/spec/workers/concerns/cronjob_queue_spec.rb
index 7dd016fc78a..0244535051f 100644
--- a/spec/workers/concerns/cronjob_queue_spec.rb
+++ b/spec/workers/concerns/cronjob_queue_spec.rb
@@ -40,6 +40,10 @@ RSpec.describe CronjobQueue do
     stub_const("AnotherWorker", another_worker)
   end
 
+  it 'sets the queue name of a worker' do
+    expect(worker.sidekiq_options['queue'].to_s).to eq('cronjob:dummy')
+  end
+
   it 'disables retrying of failed jobs' do
     expect(worker.sidekiq_options['retry']).to eq(false)
   end
diff --git a/spec/workers/concerns/gitlab/github_import/queue_spec.rb b/spec/workers/concerns/gitlab/github_import/queue_spec.rb
new file mode 100644
index 00000000000..beca221b593
--- /dev/null
+++ b/spec/workers/concerns/gitlab/github_import/queue_spec.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::GithubImport::Queue do
+  it 'sets the Sidekiq options for the worker' do
+    worker = Class.new do
+      def self.name
+        'DummyWorker'
+      end
+
+      include ApplicationWorker
+      include Gitlab::GithubImport::Queue
+    end
+
+    expect(worker.sidekiq_options['queue']).to eq('github_importer:dummy')
+  end
+end
diff --git a/spec/workers/concerns/pipeline_background_queue_spec.rb b/spec/workers/concerns/pipeline_background_queue_spec.rb
new file mode 100644
index 00000000000..77c7e7440c5
--- /dev/null
+++ b/spec/workers/concerns/pipeline_background_queue_spec.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe PipelineBackgroundQueue do
+  let(:worker) do
+    Class.new do
+      def self.name
+        'DummyWorker'
+      end
+
+      include ApplicationWorker
+      include PipelineBackgroundQueue
+    end
+  end
+
+  it 'sets a default object storage queue automatically' do
+    expect(worker.sidekiq_options['queue'])
+      .to eq 'pipeline_background:dummy'
+  end
+end
diff --git a/spec/workers/concerns/pipeline_queue_spec.rb b/spec/workers/concerns/pipeline_queue_spec.rb
new file mode 100644
index 00000000000..6c1ac2052e4
--- /dev/null
+++ b/spec/workers/concerns/pipeline_queue_spec.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe PipelineQueue do
+  let(:worker) do
+    Class.new do
+      def self.name
+        'DummyWorker'
+      end
+
+      include ApplicationWorker
+      include PipelineQueue
+    end
+  end
+
+  it 'sets a default pipelines queue automatically' do
+    expect(worker.sidekiq_options['queue'])
+      .to eq 'pipeline_default:dummy'
+  end
+end
diff --git a/spec/workers/concerns/repository_check_queue_spec.rb b/spec/workers/concerns/repository_check_queue_spec.rb
index 08ac73aac7b..ae377c09b37 100644
--- a/spec/workers/concerns/repository_check_queue_spec.rb
+++ b/spec/workers/concerns/repository_check_queue_spec.rb
@@ -14,6 +14,10 @@ RSpec.describe RepositoryCheckQueue do
     end
   end
 
+  it 'sets the queue name of a worker' do
+    expect(worker.sidekiq_options['queue'].to_s).to eq('repository_check:dummy')
+  end
+
   it 'disables retrying of failed jobs' do
     expect(worker.sidekiq_options['retry']).to eq(false)
   end
diff --git a/spec/workers/concerns/waitable_worker_spec.rb b/spec/workers/concerns/waitable_worker_spec.rb
index 2df5b60deaf..bf156c3b8cb 100644
--- a/spec/workers/concerns/waitable_worker_spec.rb
+++ b/spec/workers/concerns/waitable_worker_spec.rb
@@ -49,7 +49,8 @@ RSpec.describe WaitableWorker do
           expect(Gitlab::AppJsonLogger).to(
             receive(:info).with(a_hash_including('message' => 'running inline',
                                                  'class' => 'Gitlab::Foo::Bar::DummyWorker',
-                                                 'job_status' => 'running'))
+                                                 'job_status' => 'running',
+                                                 'queue' => 'foo_bar_dummy'))
                           .once)
 
           worker.bulk_perform_and_wait(args_list)
diff --git a/spec/workers/ssh_keys/expired_notification_worker_spec.rb b/spec/workers/ssh_keys/expired_notification_worker_spec.rb
index f93d02e86c0..26d9460d73e 100644
--- a/spec/workers/ssh_keys/expired_notification_worker_spec.rb
+++ b/spec/workers/ssh_keys/expired_notification_worker_spec.rb
@@ -7,6 +7,7 @@ RSpec.describe SshKeys::ExpiredNotificationWorker, type: :worker do
 
   it 'uses a cronjob queue' do
     expect(worker.sidekiq_options_hash).to include(
+      'queue' => 'cronjob:ssh_keys_expired_notification',
       'queue_namespace' => :cronjob
     )
   end
diff --git a/spec/workers/ssh_keys/expiring_soon_notification_worker_spec.rb b/spec/workers/ssh_keys/expiring_soon_notification_worker_spec.rb
index ed6701532a5..e907d035020 100644
--- a/spec/workers/ssh_keys/expiring_soon_notification_worker_spec.rb
+++ b/spec/workers/ssh_keys/expiring_soon_notification_worker_spec.rb
@@ -7,6 +7,7 @@ RSpec.describe SshKeys::ExpiringSoonNotificationWorker, type: :worker do
 
   it 'uses a cronjob queue' do
     expect(worker.sidekiq_options_hash).to include(
+      'queue' => 'cronjob:ssh_keys_expiring_soon_notification',
       'queue_namespace' => :cronjob
     )
   end