Add RepositoryCheck::DispatchWorker to start worker per shardtc-repo-check-per-shard

The RepositoryCheck::DispatchWorker will start a
RepositoryCheck::BatchWorker for each healthy shard.

Closes gitlab-org/gitlab-ce#48042

7 files changed, 72 insertions, 11 deletions

diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 026f756582d..d06f51b1828 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -11,7 +11,7 @@
 - cronjob:remove_old_web_hook_logs
 - cronjob:remove_unreferenced_lfs_objects
 - cronjob:repository_archive_cache
-- cronjob:repository_check_batch
+- cronjob:repository_check_dispatch
 - cronjob:requests_profiles
 - cronjob:schedule_update_user_activity
 - cronjob:stuck_ci_jobs
@@ -71,6 +71,7 @@
 - pipeline_processing:update_head_pipeline_for_merge_request
 
 - repository_check:repository_check_clear
+- repository_check:repository_check_batch
 - repository_check:repository_check_single_repository
 
 - default
diff --git a/app/workers/repository_check/batch_worker.rb b/app/workers/repository_check/batch_worker.rb
index 72f0a9b0619..fcc12bfe09d 100644
--- a/app/workers/repository_check/batch_worker.rb
+++ b/app/workers/repository_check/batch_worker.rb
@@ -1,13 +1,18 @@
 module RepositoryCheck
   class BatchWorker
     include ApplicationWorker
-    include CronjobQueue
+    include RepositoryCheckQueue
 
     RUN_TIME = 3600
     BATCH_SIZE = 10_000
 
-    def perform
+    attr_reader :shard_name
+
+    def perform(shard_name)
+      @shard_name = shard_name
+
       return unless Gitlab::CurrentSettings.repository_checks_enabled
+      return unless Gitlab::ShardHealthCache.healthy_shard?(shard_name)
 
       start = Time.now
 
@@ -37,18 +42,22 @@ module RepositoryCheck
     end
 
     def never_checked_project_ids(batch_size)
-      Project.where(last_repository_check_at: nil)
+      projects_on_shard.where(last_repository_check_at: nil)
         .where('created_at < ?', 24.hours.ago)
         .limit(batch_size).pluck(:id)
     end
 
     def old_checked_project_ids(batch_size)
-      Project.where.not(last_repository_check_at: nil)
+      projects_on_shard.where.not(last_repository_check_at: nil)
         .where('last_repository_check_at < ?', 1.month.ago)
         .reorder(last_repository_check_at: :asc)
         .limit(batch_size).pluck(:id)
     end
 
+    def projects_on_shard
+      Project.where(repository_storage: shard_name)
+    end
+
     def try_obtain_lease(id)
       # Use a 24-hour timeout because on servers/projects where 'git fsck' is
       # super slow we definitely do not want to run it twice in parallel.
diff --git a/app/workers/repository_check/dispatch_worker.rb b/app/workers/repository_check/dispatch_worker.rb
new file mode 100644
index 00000000000..403479a8497
--- /dev/null
+++ b/app/workers/repository_check/dispatch_worker.rb
@@ -0,0 +1,15 @@
+module RepositoryCheck
+  class DispatchWorker
+    include ApplicationWorker
+    include CronjobQueue
+    include ::EachShardWorker
+
+    def perform
+      return unless Gitlab::CurrentSettings.repository_checks_enabled
+
+      each_shard do |shard_name|
+        RepositoryCheck::BatchWorker.perform_async(shard_name)
+      end
+    end
+  end
+end
diff --git a/changelogs/unreleased/tc-repo-check-per-shard.yml b/changelogs/unreleased/tc-repo-check-per-shard.yml
new file mode 100644
index 00000000000..e402b72ba4b
--- /dev/null
+++ b/changelogs/unreleased/tc-repo-check-per-shard.yml
@@ -0,0 +1,5 @@
+---
+title: Run repository checks parallel for each shard
+merge_request: 20179
+author:
+type: added
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 3d3448cb4d6..3236b2c2f15 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -279,7 +279,7 @@ Settings.cron_jobs['expire_build_artifacts_worker']['cron'] ||= '50 * * * *'
 Settings.cron_jobs['expire_build_artifacts_worker']['job_class'] = 'ExpireBuildArtifactsWorker'
 Settings.cron_jobs['repository_check_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['repository_check_worker']['cron'] ||= '20 * * * *'
-Settings.cron_jobs['repository_check_worker']['job_class'] = 'RepositoryCheck::BatchWorker'
+Settings.cron_jobs['repository_check_worker']['job_class'] = 'RepositoryCheck::DispatchWorker'
 Settings.cron_jobs['admin_email_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['admin_email_worker']['cron'] ||= '0 0 * * 0'
 Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker'
diff --git a/spec/workers/repository_check/batch_worker_spec.rb b/spec/workers/repository_check/batch_worker_spec.rb
index 6cd27d2fafb..a070e6b1133 100644
--- a/spec/workers/repository_check/batch_worker_spec.rb
+++ b/spec/workers/repository_check/batch_worker_spec.rb
@@ -1,14 +1,19 @@
 require 'spec_helper'
 
 describe RepositoryCheck::BatchWorker do
+  let(:shard_name) { 'default' }
   subject { described_class.new }
 
+  before do
+    Gitlab::ShardHealthCache.update([shard_name])
+  end
+
   it 'prefers projects that have never been checked' do
     projects = create_list(:project, 3, created_at: 1.week.ago)
     projects[0].update_column(:last_repository_check_at, 4.months.ago)
     projects[2].update_column(:last_repository_check_at, 3.months.ago)
 
-    expect(subject.perform).to eq(projects.values_at(1, 0, 2).map(&:id))
+    expect(subject.perform(shard_name)).to eq(projects.values_at(1, 0, 2).map(&:id))
   end
 
   it 'sorts projects by last_repository_check_at' do
@@ -17,7 +22,7 @@ describe RepositoryCheck::BatchWorker do
     projects[1].update_column(:last_repository_check_at, 4.months.ago)
     projects[2].update_column(:last_repository_check_at, 3.months.ago)
 
-    expect(subject.perform).to eq(projects.values_at(1, 2, 0).map(&:id))
+    expect(subject.perform(shard_name)).to eq(projects.values_at(1, 2, 0).map(&:id))
   end
 
   it 'excludes projects that were checked recently' do
@@ -26,7 +31,7 @@ describe RepositoryCheck::BatchWorker do
     projects[1].update_column(:last_repository_check_at, 2.months.ago)
     projects[2].update_column(:last_repository_check_at, 3.days.ago)
 
-    expect(subject.perform).to eq([projects[1].id])
+    expect(subject.perform(shard_name)).to eq([projects[1].id])
   end
 
   it 'does nothing when repository checks are disabled' do
@@ -34,13 +39,20 @@ describe RepositoryCheck::BatchWorker do
 
     stub_application_setting(repository_checks_enabled: false)
 
-    expect(subject.perform).to eq(nil)
+    expect(subject.perform(shard_name)).to eq(nil)
+  end
+
+  it 'does nothing when shard is unhealthy' do
+    shard_name = 'broken'
+    create(:project, created_at: 1.week.ago, repository_storage: shard_name)
+
+    expect(subject.perform(shard_name)).to eq(nil)
   end
 
   it 'skips projects created less than 24 hours ago' do
     project = create(:project)
     project.update_column(:created_at, 23.hours.ago)
 
-    expect(subject.perform).to eq([])
+    expect(subject.perform(shard_name)).to eq([])
   end
 end
diff --git a/spec/workers/repository_check/dispatch_worker_spec.rb b/spec/workers/repository_check/dispatch_worker_spec.rb
new file mode 100644
index 00000000000..111096a4513
--- /dev/null
+++ b/spec/workers/repository_check/dispatch_worker_spec.rb
@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe RepositoryCheck::DispatchWorker do
+  subject { described_class.new }
+
+  it 'does nothing when repository checks are disabled' do
+    stub_application_setting(repository_checks_enabled: false)
+
+    expect(RepositoryCheck::BatchWorker).not_to receive(:perform_async)
+
+    subject.perform
+  end
+
+  it 'dispatches work to RepositoryCheck::BatchWorker' do
+    expect(RepositoryCheck::BatchWorker).to receive(:perform_async)
+
+    subject.perform
+  end
+end