diff options
author | James Lopez <james@jameslopez.es> | 2017-03-24 12:08:34 +0100 |
---|---|---|
committer | James Lopez <james@jameslopez.es> | 2017-04-05 16:11:51 +0200 |
commit | 58371efbb0bd051d3a82f82acac98ad4692efeb4 (patch) | |
tree | 226bfb1a920a6344d6efb3b0357473412aa9dade /app/workers | |
parent | 7196adaaa4ce5aa259d285dbca6aa98f4deb046b (diff) | |
download | gitlab-ce-58371efbb0bd051d3a82f82acac98ad4692efeb4.tar.gz |
Periodically mark projects that are stuck in importing as failed
Adds import jid to projects
Refactor middleware to set custom expiration time via sidekiq options
Add completed_jids option to sidekiq status and a few other changes
Diffstat (limited to 'app/workers')
-rw-r--r-- | app/workers/repository_import_worker.rb | 4 | ||||
-rw-r--r-- | app/workers/stuck_import_jobs_worker.rb | 50 |
2 files changed, 53 insertions, 1 deletions
diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 68a6fd76e70..b33ba2ed7c1 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -2,6 +2,8 @@ class RepositoryImportWorker include Sidekiq::Worker include DedicatedSidekiqQueue + sidekiq_options status_expiration: StuckImportJobsWorker::IMPORT_EXPIRATION + attr_accessor :project, :current_user def perform(project_id) @@ -12,7 +14,7 @@ class RepositoryImportWorker import_url: @project.import_url, path: @project.path_with_namespace) - project.update_column(:import_error, nil) + project.update_columns(import_jid: self.jid, import_error: nil) result = Projects::ImportService.new(project, current_user).execute diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb new file mode 100644 index 00000000000..c7871d99492 --- /dev/null +++ b/app/workers/stuck_import_jobs_worker.rb @@ -0,0 +1,50 @@ +class StuckImportJobsWorker + include Sidekiq::Worker + include CronjobQueue + + EXCLUSIVE_LEASE_KEY = 'fail_stuck_imports_worker_lease'.freeze + IMPORT_EXPIRATION = 15.hours.to_i + + def perform + return unless try_obtain_lease + + stuck_projects.find_in_batches(batch_size: 500) do |group| + jids = group.map(&:import_jid) + + # Find the jobs that aren't currently running or that exceeded the threshold. + completed_jids = Gitlab::SidekiqStatus.completed_jids(jids) + + if completed_jids.any? + completed_ids = group.select { |project| completed_jids.include?(project.import_jid) }.map(&:id) + + fail_batch!(completed_jids, completed_ids) + end + end + + remove_lease + end + + private + + def stuck_projects + Project.select('id, import_jid').with_import_status(:started).where.not(import_jid: nil) + end + + def fail_batch!(completed_jids, completed_ids) + Project.where(id: completed_ids).update_all(import_status: 'failed', import_error: error_message) + + Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.join(', ')}") + end + + def error_message + "Import timed out. Import took longer than #{IMPORT_EXPIRATION} seconds" + end + + def try_obtain_lease + @uuid = Gitlab::ExclusiveLease.new(EXCLUSIVE_LEASE_KEY, timeout: 30.minutes).try_obtain + end + + def remove_lease + Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid) + end +end |