diff options
Diffstat (limited to 'app/workers/concerns')
6 files changed, 87 insertions, 6 deletions
diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb index 63c1ba8e699..575cd4862b0 100644 --- a/app/workers/concerns/gitlab/github_import/object_importer.rb +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -15,17 +15,25 @@ module Gitlab feature_category :importers worker_has_external_dependencies! + + def logger + @logger ||= Gitlab::Import::Logger.build + end end # project - An instance of `Project` to import the data into. # client - An instance of `Gitlab::GithubImport::Client` # hash - A Hash containing the details of the object to import. def import(project, client, hash) - object = representation_class.from_json_hash(hash) + info(project.id, message: 'starting importer') + object = representation_class.from_json_hash(hash) importer_class.new(object, project, client).execute counter.increment + info(project.id, message: 'importer finished') + rescue => e + error(project.id, e) end def counter @@ -52,6 +60,35 @@ module Gitlab def counter_description raise NotImplementedError end + + private + + def info(project_id, extra = {}) + logger.info(log_attributes(project_id, extra)) + end + + def error(project_id, exception) + logger.error( + log_attributes( + project_id, + message: 'importer failed', + 'error.message': exception.message + ) + ) + + Gitlab::ErrorTracking.track_and_raise_exception( + exception, + log_attributes(project_id) + ) + end + + def log_attributes(project_id, extra = {}) + extra.merge( + import_source: :github, + project_id: project_id, + importer: importer_class.name + ) + end end end end diff --git a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb index 1c6413674a0..eb1af0869bd 100644 --- a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb +++ b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb @@ -6,7 +6,7 @@ module Gitlab # importing GitHub projects. module ReschedulingMethods # project_id - The ID of the GitLab project to import the note into. - # hash - A Hash containing the details of the GitHub object to imoprt. + # hash - A Hash containing the details of the GitHub object to import. # notify_key - The Redis key to notify upon completion, if any. # rubocop: disable CodeReuse/ActiveRecord def perform(project_id, hash, notify_key = nil) diff --git a/app/workers/concerns/gitlab/github_import/stage_methods.rb b/app/workers/concerns/gitlab/github_import/stage_methods.rb index e2dee315cde..e5985fb94da 100644 --- a/app/workers/concerns/gitlab/github_import/stage_methods.rb +++ b/app/workers/concerns/gitlab/github_import/stage_methods.rb @@ -5,11 +5,17 @@ module Gitlab module StageMethods # project_id - The ID of the GitLab project to import the data into. def perform(project_id) + info(project_id, message: 'starting stage') + return unless (project = find_project(project_id)) client = GithubImport.new_client_for(project) try_import(client, project) + + info(project_id, message: 'stage finished') + rescue => e + error(project_id, e) end # client - An instance of Gitlab::GithubImport::Client. @@ -27,6 +33,39 @@ module Gitlab Project.joins_import_state.where(import_state: { status: :started }).find_by(id: id) end # rubocop: enable CodeReuse/ActiveRecord + + private + + def info(project_id, extra = {}) + logger.info(log_attributes(project_id, extra)) + end + + def error(project_id, exception) + logger.error( + log_attributes( + project_id, + message: 'stage failed', + 'error.message': exception.message + ) + ) + + Gitlab::ErrorTracking.track_and_raise_exception( + exception, + log_attributes(project_id) + ) + end + + def log_attributes(project_id, extra = {}) + extra.merge( + import_source: :github, + project_id: project_id, + import_stage: self.class.name + ) + end + + def logger + @logger ||= Gitlab::Import::Logger.build + end end end end diff --git a/app/workers/concerns/limited_capacity/worker.rb b/app/workers/concerns/limited_capacity/worker.rb index b5a97e49300..9dd8d942146 100644 --- a/app/workers/concerns/limited_capacity/worker.rb +++ b/app/workers/concerns/limited_capacity/worker.rb @@ -73,7 +73,7 @@ module LimitedCapacity raise ensure job_tracker.remove(jid) - report_prometheus_metrics + report_prometheus_metrics(*args) re_enqueue(*args) unless exception end diff --git a/app/workers/concerns/reenqueuer.rb b/app/workers/concerns/reenqueuer.rb index 6f399b6d90b..641ca691868 100644 --- a/app/workers/concerns/reenqueuer.rb +++ b/app/workers/concerns/reenqueuer.rb @@ -37,6 +37,7 @@ module Reenqueuer include ReenqueuerSleeper sidekiq_options retry: false + deduplicate :none end def perform(*args) @@ -52,7 +53,11 @@ module Reenqueuer private def reenqueue(*args) - self.class.perform_async(*args) if yield + result = yield + + self.class.perform_async(*args) if result + + result end # Override as needed diff --git a/app/workers/concerns/worker_context.rb b/app/workers/concerns/worker_context.rb index f2ff3ecfb6b..6acb9acceeb 100644 --- a/app/workers/concerns/worker_context.rb +++ b/app/workers/concerns/worker_context.rb @@ -5,7 +5,7 @@ module WorkerContext class_methods do def worker_context(attributes) - @worker_context = Gitlab::ApplicationContext.new(attributes) + @worker_context = Gitlab::ApplicationContext.new(**attributes) end def get_worker_context @@ -60,6 +60,6 @@ module WorkerContext end def with_context(context, &block) - Gitlab::ApplicationContext.new(context).use { yield(**context) } + Gitlab::ApplicationContext.new(**context).use { yield(**context) } end end |