diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-04-21 12:21:08 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-04-21 12:21:08 +0000 |
commit | e7198b914bf1d6594909e35d3d00d0a0b260f250 (patch) | |
tree | 6fe636fedeae9c679839d5bf27dc091af037e765 /app/workers | |
parent | 3b80f22aba42e3e424de5c3dd15cc11f96aaac65 (diff) | |
download | gitlab-ce-e7198b914bf1d6594909e35d3d00d0a0b260f250.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app/workers')
-rw-r--r-- | app/workers/all_queues.yml | 9 | ||||
-rw-r--r-- | app/workers/gitlab/phabricator_import/base_worker.rb | 81 | ||||
-rw-r--r-- | app/workers/gitlab/phabricator_import/import_tasks_worker.rb | 17 |
3 files changed, 0 insertions, 107 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index f47d5da95f0..582f5037e48 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -3081,15 +3081,6 @@ :weight: 1 :idempotent: true :tags: [] -- :name: phabricator_import_import_tasks - :worker_name: Gitlab::PhabricatorImport::ImportTasksWorker - :feature_category: :importers - :has_external_dependencies: false - :urgency: :low - :resource_boundary: :unknown - :weight: 1 - :idempotent: false - :tags: [] - :name: post_receive :worker_name: PostReceive :feature_category: :source_code_management diff --git a/app/workers/gitlab/phabricator_import/base_worker.rb b/app/workers/gitlab/phabricator_import/base_worker.rb deleted file mode 100644 index 2dc4855f854..00000000000 --- a/app/workers/gitlab/phabricator_import/base_worker.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true - -# All workers within a Phabricator import should inherit from this worker and -# implement the `#import` method. The jobs should then be scheduled using the -# `.schedule` class method instead of `.perform_async` -# -# Doing this makes sure that only one job of that type is running at the same time -# for a certain project. This will avoid deadlocks. When a job is already running -# we'll wait for it for 10 times 5 seconds to restart. If the running job hasn't -# finished, by then, we'll retry in 30 seconds. -# -# It also makes sure that we keep the import state of the project up to date: -# - It keeps track of the jobs so we know how many jobs are running for the -# project -# - It refreshes the import jid, so it doesn't get cleaned up by the -# `Gitlab::Import::StuckProjectImportJobsWorker` -# - It marks the import as failed if a job failed to many times -# - It marks the import as finished when all remaining jobs are done -module Gitlab - module PhabricatorImport - class BaseWorker # rubocop:disable Scalability/IdempotentWorker - include WorkerAttributes - include Gitlab::ExclusiveLeaseHelpers - - feature_category :importers - - class << self - def schedule(project_id, *args) - perform_async(project_id, *args) - add_job(project_id) - end - - def add_job(project_id) - worker_state(project_id).add_job - end - - def remove_job(project_id) - worker_state(project_id).remove_job - end - - def worker_state(project_id) - Gitlab::PhabricatorImport::WorkerState.new(project_id) - end - end - - def perform(project_id, *args) - in_lock("#{self.class.name.underscore}/#{project_id}/#{args}", ttl: 2.hours, sleep_sec: 5.seconds) do - project = Project.find_by_id(project_id) - next unless project - - # Bail if the import job already failed - next unless project.import_state&.in_progress? - - project.import_state.refresh_jid_expiration - - import(project, *args) - - # If this is the last running job, finish the import - project.after_import if self.class.worker_state(project_id).running_count < 2 - - self.class.remove_job(project_id) - end - rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError - # Reschedule a job if there was already a running one - # Running them at the same time could cause a deadlock updating the same - # resource - self.class.perform_in(30.seconds, project_id, *args) - end - - private - - def import(project, *args) - importer_class.new(project, *args).execute - end - - def importer_class - raise NotImplementedError, "Implement `#{__method__}` on #{self.class}" - end - end - end -end diff --git a/app/workers/gitlab/phabricator_import/import_tasks_worker.rb b/app/workers/gitlab/phabricator_import/import_tasks_worker.rb deleted file mode 100644 index f650681fc2f..00000000000 --- a/app/workers/gitlab/phabricator_import/import_tasks_worker.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module PhabricatorImport - class ImportTasksWorker < BaseWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker - - data_consistency :always - - sidekiq_options retry: 3 - include ProjectImportOptions # This marks the project as failed after too many tries - - def importer_class - Gitlab::PhabricatorImport::Issues::Importer - end - end - end -end |