diff options
author | Toon Claes <toon@gitlab.com> | 2018-06-26 15:12:29 +0200 |
---|---|---|
committer | Toon Claes <toon@gitlab.com> | 2018-06-27 21:43:23 +0200 |
commit | 95ac8b0e1d0385429f5a09edf0dc908346fdd3a7 (patch) | |
tree | 78501c423558294caf5a5dc3d31ecb460e3f9d2b /app/workers/repository_check | |
parent | 48901bdecfe30fd201c01a608fdc3b35e4f70e08 (diff) | |
download | gitlab-ce-95ac8b0e1d0385429f5a09edf0dc908346fdd3a7.tar.gz |
Add RepositoryCheck::DispatchWorker to start worker per shard
The RepositoryCheck::DispatchWorker will start a
RepositoryCheck::BatchWorker for each healthy shard.
Closes gitlab-org/gitlab-ce#48042
Diffstat (limited to 'app/workers/repository_check')
-rw-r--r-- | app/workers/repository_check/batch_worker.rb | 17 | ||||
-rw-r--r-- | app/workers/repository_check/dispatch_worker.rb | 15 |
2 files changed, 28 insertions, 4 deletions
diff --git a/app/workers/repository_check/batch_worker.rb b/app/workers/repository_check/batch_worker.rb index 898bca976ec..051382a08a9 100644 --- a/app/workers/repository_check/batch_worker.rb +++ b/app/workers/repository_check/batch_worker.rb @@ -3,13 +3,18 @@ module RepositoryCheck class BatchWorker include ApplicationWorker - include CronjobQueue + include RepositoryCheckQueue RUN_TIME = 3600 BATCH_SIZE = 10_000 - def perform + attr_reader :shard_name + + def perform(shard_name) + @shard_name = shard_name + return unless Gitlab::CurrentSettings.repository_checks_enabled + return unless Gitlab::ShardHealthCache.healthy_shard?(shard_name) start = Time.now @@ -39,18 +44,22 @@ module RepositoryCheck end def never_checked_project_ids(batch_size) - Project.where(last_repository_check_at: nil) + projects_on_shard.where(last_repository_check_at: nil) .where('created_at < ?', 24.hours.ago) .limit(batch_size).pluck(:id) end def old_checked_project_ids(batch_size) - Project.where.not(last_repository_check_at: nil) + projects_on_shard.where.not(last_repository_check_at: nil) .where('last_repository_check_at < ?', 1.month.ago) .reorder(last_repository_check_at: :asc) .limit(batch_size).pluck(:id) end + def projects_on_shard + Project.where(repository_storage: shard_name) + end + def try_obtain_lease(id) # Use a 24-hour timeout because on servers/projects where 'git fsck' is # super slow we definitely do not want to run it twice in parallel. diff --git a/app/workers/repository_check/dispatch_worker.rb b/app/workers/repository_check/dispatch_worker.rb new file mode 100644 index 00000000000..403479a8497 --- /dev/null +++ b/app/workers/repository_check/dispatch_worker.rb @@ -0,0 +1,15 @@ +module RepositoryCheck + class DispatchWorker + include ApplicationWorker + include CronjobQueue + include ::EachShardWorker + + def perform + return unless Gitlab::CurrentSettings.repository_checks_enabled + + each_shard do |shard_name| + RepositoryCheck::BatchWorker.perform_async(shard_name) + end + end + end +end |