diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-20 12:26:25 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-20 12:26:25 +0000 |
commit | a09983ae35713f5a2bbb100981116d31ce99826e (patch) | |
tree | 2ee2af7bd104d57086db360a7e6d8c9d5d43667a /app/workers | |
parent | 18c5ab32b738c0b6ecb4d0df3994000482f34bd8 (diff) | |
download | gitlab-ce-a09983ae35713f5a2bbb100981116d31ce99826e.tar.gz |
Add latest changes from gitlab-org/gitlab@13-2-stable-ee
Diffstat (limited to 'app/workers')
29 files changed, 355 insertions, 217 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 3baa2166812..5148772c881 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -11,6 +11,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: authorized_project_update:authorized_project_update_project_group_link_create + :feature_category: :authentication_and_authorization + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: authorized_project_update:authorized_project_update_user_refresh_over_user_range :feature_category: :authentication_and_authorization :has_external_dependencies: @@ -227,6 +235,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: cronjob:partition_creation + :feature_category: :database + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:personal_access_tokens_expiring :feature_category: :authentication_and_authorization :has_external_dependencies: @@ -331,15 +347,15 @@ :weight: 1 :idempotent: :tags: [] -- :name: cronjob:stuck_import_jobs - :feature_category: :importers +- :name: cronjob:stuck_merge_jobs + :feature_category: :source_code_management :has_external_dependencies: :urgency: :low - :resource_boundary: :cpu + :resource_boundary: :unknown :weight: 1 :idempotent: :tags: [] -- :name: cronjob:stuck_merge_jobs +- :name: cronjob:trending_projects :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -347,13 +363,13 @@ :weight: 1 :idempotent: :tags: [] -- :name: cronjob:trending_projects - :feature_category: :source_code_management +- :name: cronjob:update_container_registry_info + :feature_category: :container_registry :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 - :idempotent: + :idempotent: true :tags: [] - :name: cronjob:users_create_statistics :feature_category: :users @@ -675,6 +691,14 @@ :weight: 2 :idempotent: true :tags: [] +- :name: incident_management:incident_management_pager_duty_process_incident + :feature_category: :incident_management + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 2 + :idempotent: + :tags: [] - :name: incident_management:incident_management_process_alert :feature_category: :incident_management :has_external_dependencies: @@ -771,14 +795,6 @@ :weight: 2 :idempotent: :tags: [] -- :name: notifications:new_release - :feature_category: :release_orchestration - :has_external_dependencies: - :urgency: :low - :resource_boundary: :unknown - :weight: 2 - :idempotent: - :tags: [] - :name: object_pool:object_pool_create :feature_category: :gitaly :has_external_dependencies: @@ -827,6 +843,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: package_repositories:packages_nuget_extraction + :feature_category: :package_registry + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] - :name: pipeline_background:archive_trace :feature_category: :continuous_integration :has_external_dependencies: @@ -859,6 +883,22 @@ :weight: 1 :idempotent: true :tags: [] +- :name: pipeline_background:ci_pipeline_success_unlock_artifacts + :feature_category: :continuous_integration + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] +- :name: pipeline_background:ci_ref_delete_unlock_artifacts + :feature_category: :continuous_integration + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: pipeline_cache:expire_job_cache :feature_category: :continuous_integration :has_external_dependencies: @@ -970,7 +1010,8 @@ :resource_boundary: :cpu :weight: 5 :idempotent: - :tags: [] + :tags: + - :requires_disk_io - :name: pipeline_processing:build_queue :feature_category: :continuous_integration :has_external_dependencies: @@ -1025,7 +1066,7 @@ :urgency: :high :resource_boundary: :unknown :weight: 5 - :idempotent: + :idempotent: true :tags: [] - :name: pipeline_processing:stage_update :feature_category: :continuous_integration @@ -1107,6 +1148,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: unassign_issuables:members_destroyer_unassign_issuables + :feature_category: :authentication_and_authorization + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: update_namespace_statistics:namespaces_root_statistics :feature_category: :source_code_management :has_external_dependencies: @@ -1526,10 +1575,10 @@ - :name: project_update_repository_storage :feature_category: :gitaly :has_external_dependencies: - :urgency: :low + :urgency: :throttled :resource_boundary: :unknown :weight: 1 - :idempotent: + :idempotent: true :tags: [] - :name: prometheus_create_default_alerts :feature_category: :incident_management @@ -1635,6 +1684,14 @@ :weight: 2 :idempotent: :tags: [] +- :name: service_desk_email_receiver + :feature_category: :issue_tracking + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] - :name: system_hook_push :feature_category: :source_code_management :has_external_dependencies: diff --git a/app/workers/authorized_project_update/project_group_link_create_worker.rb b/app/workers/authorized_project_update/project_group_link_create_worker.rb new file mode 100644 index 00000000000..5fb59efaacb --- /dev/null +++ b/app/workers/authorized_project_update/project_group_link_create_worker.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module AuthorizedProjectUpdate + class ProjectGroupLinkCreateWorker + include ApplicationWorker + + feature_category :authentication_and_authorization + urgency :low + queue_namespace :authorized_project_update + + idempotent! + + def perform(project_id, group_id) + project = Project.find(project_id) + group = Group.find(group_id) + + AuthorizedProjectUpdate::ProjectGroupLinkCreateService.new(project, group) + .execute + end + end +end diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb index d38780dd08d..d0f7d65aed6 100644 --- a/app/workers/build_finished_worker.rb +++ b/app/workers/build_finished_worker.rb @@ -7,6 +7,7 @@ class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker queue_namespace :pipeline_processing urgency :high worker_resource_boundary :cpu + tags :requires_disk_io # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) diff --git a/app/workers/ci/pipeline_success_unlock_artifacts_worker.rb b/app/workers/ci/pipeline_success_unlock_artifacts_worker.rb new file mode 100644 index 00000000000..bc31876aa1d --- /dev/null +++ b/app/workers/ci/pipeline_success_unlock_artifacts_worker.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Ci + class PipelineSuccessUnlockArtifactsWorker + include ApplicationWorker + include PipelineBackgroundQueue + + idempotent! + + def perform(pipeline_id) + ::Ci::Pipeline.find_by_id(pipeline_id).try do |pipeline| + break unless pipeline.has_archive_artifacts? + + ::Ci::UnlockArtifactsService + .new(pipeline.project, pipeline.user) + .execute(pipeline.ci_ref, pipeline) + end + end + end +end diff --git a/app/workers/ci/ref_delete_unlock_artifacts_worker.rb b/app/workers/ci/ref_delete_unlock_artifacts_worker.rb new file mode 100644 index 00000000000..3b4a6fcf630 --- /dev/null +++ b/app/workers/ci/ref_delete_unlock_artifacts_worker.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Ci + class RefDeleteUnlockArtifactsWorker + include ApplicationWorker + include PipelineBackgroundQueue + + idempotent! + + def perform(project_id, user_id, ref_path) + ::Project.find_by_id(project_id).try do |project| + ::User.find_by_id(user_id).try do |user| + ::Ci::Ref.find_by_ref_path(ref_path).try do |ci_ref| + ::Ci::UnlockArtifactsService + .new(project, user) + .execute(ci_ref) + end + end + end + end + end +end diff --git a/app/workers/concerns/project_export_options.rb b/app/workers/concerns/project_export_options.rb deleted file mode 100644 index e9318c1ba43..00000000000 --- a/app/workers/concerns/project_export_options.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module ProjectExportOptions - extend ActiveSupport::Concern - - EXPORT_RETRY_COUNT = 3 - - included do - sidekiq_options retry: EXPORT_RETRY_COUNT, status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION - - # We mark the project export as failed once we have exhausted all retries - sidekiq_retries_exhausted do |job| - project = Project.find(job['args'][1]) - # rubocop: disable CodeReuse/ActiveRecord - job = project.export_jobs.find_by(jid: job["jid"]) - # rubocop: enable CodeReuse/ActiveRecord - - if job&.fail_op - Sidekiq.logger.info "Job #{job['jid']} for project #{project.id} has been set to failed state" - else - Sidekiq.logger.error "Failed to set Job #{job['jid']} for project #{project.id} to failed state" - end - end - end -end diff --git a/app/workers/concerns/reenqueuer.rb b/app/workers/concerns/reenqueuer.rb index 5cc13e490d8..bf6f6546c03 100644 --- a/app/workers/concerns/reenqueuer.rb +++ b/app/workers/concerns/reenqueuer.rb @@ -60,8 +60,6 @@ module Reenqueuer 5.seconds end - # We intend to get rid of sleep: - # https://gitlab.com/gitlab-org/gitlab/issues/121697 module ReenqueuerSleeper # The block will run, and then sleep until the minimum duration. Returns the # block's return value. @@ -73,7 +71,7 @@ module Reenqueuer # end # def ensure_minimum_duration(minimum_duration) - start_time = Time.now + start_time = Time.current result = yield @@ -95,7 +93,7 @@ module Reenqueuer end def elapsed_time(start_time) - Time.now - start_time + Time.current - start_time end end end diff --git a/app/workers/concerns/worker_attributes.rb b/app/workers/concerns/worker_attributes.rb index b19217b15de..bb6192166b4 100644 --- a/app/workers/concerns/worker_attributes.rb +++ b/app/workers/concerns/worker_attributes.rb @@ -2,6 +2,7 @@ module WorkerAttributes extend ActiveSupport::Concern + include Gitlab::ClassAttributes # Resource boundaries that workers can declare through the # `resource_boundary` attribute @@ -30,24 +31,24 @@ module WorkerAttributes }.stringify_keys.freeze class_methods do - def feature_category(value) + def feature_category(value, *extras) raise "Invalid category. Use `feature_category_not_owned!` to mark a worker as not owned" if value == :not_owned - worker_attributes[:feature_category] = value + class_attributes[:feature_category] = value end # Special case: mark this work as not associated with a feature category # this should be used for cross-cutting concerns, such as mailer workers. def feature_category_not_owned! - worker_attributes[:feature_category] = :not_owned + class_attributes[:feature_category] = :not_owned end def get_feature_category - get_worker_attribute(:feature_category) + get_class_attribute(:feature_category) end def feature_category_not_owned? - get_worker_attribute(:feature_category) == :not_owned + get_feature_category == :not_owned end # This should be set to :high for jobs that need to be run @@ -61,97 +62,76 @@ module WorkerAttributes def urgency(urgency) raise "Invalid urgency: #{urgency}" unless VALID_URGENCIES.include?(urgency) - worker_attributes[:urgency] = urgency + class_attributes[:urgency] = urgency end def get_urgency - worker_attributes[:urgency] || :low + class_attributes[:urgency] || :low end # Set this attribute on a job when it will call to services outside of the # application, such as 3rd party applications, other k8s clusters etc See - # doc/development/sidekiq_style_guide.md#Jobs-with-External-Dependencies for + # doc/development/sidekiq_style_guide.md#jobs-with-external-dependencies for # details def worker_has_external_dependencies! - worker_attributes[:external_dependencies] = true + class_attributes[:external_dependencies] = true end # Returns a truthy value if the worker has external dependencies. - # See doc/development/sidekiq_style_guide.md#Jobs-with-External-Dependencies + # See doc/development/sidekiq_style_guide.md#jobs-with-external-dependencies # for details def worker_has_external_dependencies? - worker_attributes[:external_dependencies] + class_attributes[:external_dependencies] end def worker_resource_boundary(boundary) raise "Invalid boundary" unless VALID_RESOURCE_BOUNDARIES.include? boundary - worker_attributes[:resource_boundary] = boundary + class_attributes[:resource_boundary] = boundary end def get_worker_resource_boundary - worker_attributes[:resource_boundary] || :unknown + class_attributes[:resource_boundary] || :unknown end def idempotent! - worker_attributes[:idempotent] = true + class_attributes[:idempotent] = true end def idempotent? - worker_attributes[:idempotent] + class_attributes[:idempotent] end def weight(value) - worker_attributes[:weight] = value + class_attributes[:weight] = value end def get_weight - worker_attributes[:weight] || + class_attributes[:weight] || NAMESPACE_WEIGHTS[queue_namespace] || 1 end def tags(*values) - worker_attributes[:tags] = values + class_attributes[:tags] = values end def get_tags - Array(worker_attributes[:tags]) + Array(class_attributes[:tags]) end def deduplicate(strategy, options = {}) - worker_attributes[:deduplication_strategy] = strategy - worker_attributes[:deduplication_options] = options + class_attributes[:deduplication_strategy] = strategy + class_attributes[:deduplication_options] = options end def get_deduplicate_strategy - worker_attributes[:deduplication_strategy] || + class_attributes[:deduplication_strategy] || Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob::DEFAULT_STRATEGY end def get_deduplication_options - worker_attributes[:deduplication_options] || {} - end - - protected - - # Returns a worker attribute declared on this class or its parent class. - # This approach allows declared attributes to be inherited by - # child classes. - def get_worker_attribute(name) - worker_attributes[name] || superclass_worker_attributes(name) - end - - private - - def worker_attributes - @attributes ||= {} - end - - def superclass_worker_attributes(name) - return unless superclass.include? WorkerAttributes - - superclass.get_worker_attribute(name) + class_attributes[:deduplication_options] || {} end end end diff --git a/app/workers/delete_merged_branches_worker.rb b/app/workers/delete_merged_branches_worker.rb index ab3d42e5384..8d7026e2d1e 100644 --- a/app/workers/delete_merged_branches_worker.rb +++ b/app/workers/delete_merged_branches_worker.rb @@ -17,7 +17,6 @@ class DeleteMergedBranchesWorker # rubocop:disable Scalability/IdempotentWorker begin ::Branches::DeleteMergedService.new(project, user).execute rescue Gitlab::Access::AccessDeniedError - return end end end diff --git a/app/workers/gitlab/jira_import/import_issue_worker.rb b/app/workers/gitlab/jira_import/import_issue_worker.rb index 7709d2ec31b..d1ceda4fd6a 100644 --- a/app/workers/gitlab/jira_import/import_issue_worker.rb +++ b/app/workers/gitlab/jira_import/import_issue_worker.rb @@ -62,7 +62,7 @@ module Gitlab end def build_label_attrs(issue_id, label_id) - time = Time.now + time = Time.current { label_id: label_id, target_id: issue_id, diff --git a/app/workers/group_export_worker.rb b/app/workers/group_export_worker.rb index 6fd977e43d8..e22b691d35e 100644 --- a/app/workers/group_export_worker.rb +++ b/app/workers/group_export_worker.rb @@ -6,6 +6,7 @@ class GroupExportWorker # rubocop:disable Scalability/IdempotentWorker feature_category :importers loggable_arguments 2 + sidekiq_options retry: false def perform(current_user_id, group_id, params = {}) current_user = User.find(current_user_id) diff --git a/app/workers/incident_management/pager_duty/process_incident_worker.rb b/app/workers/incident_management/pager_duty/process_incident_worker.rb new file mode 100644 index 00000000000..3f378b012a1 --- /dev/null +++ b/app/workers/incident_management/pager_duty/process_incident_worker.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module IncidentManagement + module PagerDuty + class ProcessIncidentWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + queue_namespace :incident_management + feature_category :incident_management + + def perform(project_id, incident_payload) + return unless project_id + + project = find_project(project_id) + return unless project + + result = create_issue(project, incident_payload) + + log_error(result) if result.error? + end + + private + + def find_project(project_id) + Project.find_by_id(project_id) + end + + def create_issue(project, incident_payload) + ::IncidentManagement::PagerDuty::CreateIncidentIssueService + .new(project, incident_payload) + .execute + end + + def log_error(result) + Gitlab::AppLogger.warn( + message: 'Cannot create issue for PagerDuty incident', + issue_errors: result.message + ) + end + end + end +end diff --git a/app/workers/incident_management/process_alert_worker.rb b/app/workers/incident_management/process_alert_worker.rb index 0af34fa35d5..bc23dbda693 100644 --- a/app/workers/incident_management/process_alert_worker.rb +++ b/app/workers/incident_management/process_alert_worker.rb @@ -7,39 +7,45 @@ module IncidentManagement queue_namespace :incident_management feature_category :incident_management - def perform(project_id, alert_payload, am_alert_id = nil) - project = find_project(project_id) - return unless project + # `project_id` and `alert_payload` are deprecated and can be removed + # starting from 14.0 release + # https://gitlab.com/gitlab-org/gitlab/-/issues/224500 + def perform(_project_id = nil, _alert_payload = nil, alert_id = nil) + return unless alert_id - new_issue = create_issue(project, alert_payload) - return unless am_alert_id && new_issue&.persisted? + alert = find_alert(alert_id) + return unless alert + + new_issue = create_issue_for(alert) + return unless new_issue&.persisted? - link_issue_with_alert(am_alert_id, new_issue.id) + link_issue_with_alert(alert, new_issue.id) end private - def find_project(project_id) - Project.find_by_id(project_id) + def find_alert(alert_id) + AlertManagement::Alert.find_by_id(alert_id) + end + + def parsed_payload(alert) + Gitlab::Alerting::NotificationPayloadParser.call(alert.payload.to_h, alert.project) end - def create_issue(project, alert_payload) + def create_issue_for(alert) IncidentManagement::CreateIssueService - .new(project, alert_payload) + .new(alert.project, parsed_payload(alert)) .execute .dig(:issue) end - def link_issue_with_alert(alert_id, issue_id) - alert = AlertManagement::Alert.find_by_id(alert_id) - return unless alert - + def link_issue_with_alert(alert, issue_id) return if alert.update(issue_id: issue_id) Gitlab::AppLogger.warn( message: 'Cannot link an Issue with Alert', issue_id: issue_id, - alert_id: alert_id, + alert_id: alert.id, alert_errors: alert.errors.messages ) end diff --git a/app/workers/incident_management/process_prometheus_alert_worker.rb b/app/workers/incident_management/process_prometheus_alert_worker.rb index e405bc2c2d2..4b778f6a621 100644 --- a/app/workers/incident_management/process_prometheus_alert_worker.rb +++ b/app/workers/incident_management/process_prometheus_alert_worker.rb @@ -9,68 +9,13 @@ module IncidentManagement worker_resource_boundary :cpu def perform(project_id, alert_hash) - project = find_project(project_id) - return unless project - - parsed_alert = Gitlab::Alerting::Alert.new(project: project, payload: alert_hash) - event = find_prometheus_alert_event(parsed_alert) - - if event&.resolved? - issue = event.related_issues.order_created_at_desc.detect(&:opened?) - - close_issue(project, issue) - else - issue = create_issue(project, alert_hash) - - relate_issue_to_event(event, issue) - end - end - - private - - def find_project(project_id) - Project.find_by_id(project_id) - end - - def find_prometheus_alert_event(alert) - if alert.gitlab_managed? - find_gitlab_managed_event(alert) - else - find_self_managed_event(alert) - end - end - - def find_gitlab_managed_event(alert) - PrometheusAlertEvent.find_by_payload_key(alert.gitlab_fingerprint) - end - - def find_self_managed_event(alert) - SelfManagedPrometheusAlertEvent.find_by_payload_key(alert.gitlab_fingerprint) - end - - def create_issue(project, alert) - IncidentManagement::CreateIssueService - .new(project, alert) - .execute - .dig(:issue) - end - - def close_issue(project, issue) - return if issue.blank? || issue.closed? - - processed_issue = Issues::CloseService - .new(project, User.alert_bot) - .execute(issue, system_note: false) - - SystemNoteService.auto_resolve_prometheus_alert(issue, project, User.alert_bot) if processed_issue.reset.closed? - end - - def relate_issue_to_event(event, issue) - return unless event && issue - - if event.related_issues.exclude?(issue) - event.related_issues << issue - end + # no-op + # + # This worker is not scheduled anymore since + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35943 + # and will be removed completely via + # https://gitlab.com/gitlab-org/gitlab/-/issues/227146 + # in 14.0. end end end diff --git a/app/workers/members_destroyer/unassign_issuables_worker.rb b/app/workers/members_destroyer/unassign_issuables_worker.rb new file mode 100644 index 00000000000..2c17120bf48 --- /dev/null +++ b/app/workers/members_destroyer/unassign_issuables_worker.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module MembersDestroyer + class UnassignIssuablesWorker + include ApplicationWorker + + ENTITY_TYPES = %w(Group Project).freeze + + queue_namespace :unassign_issuables + feature_category :authentication_and_authorization + + idempotent! + + def perform(user_id, entity_id, entity_type) + unless ENTITY_TYPES.include?(entity_type) + logger.error( + message: "#{entity_type} is not a supported entity.", + entity_type: entity_type, + entity_id: entity_id, + user_id: user_id + ) + + return + end + + user = User.find(user_id) + entity = entity_type.constantize.find(entity_id) + + ::Members::UnassignIssuablesService.new(user, entity).execute + end + end +end diff --git a/app/workers/new_release_worker.rb b/app/workers/new_release_worker.rb deleted file mode 100644 index fa4703d10f2..00000000000 --- a/app/workers/new_release_worker.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -# TODO: Worker can be removed in 13.2: -# https://gitlab.com/gitlab-org/gitlab/-/issues/218231 -class NewReleaseWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker - - queue_namespace :notifications - feature_category :release_orchestration - weight 2 - - def perform(release_id) - release = Release.preloaded.find_by_id(release_id) - return unless release - - NotificationService.new.send_new_release_notifications(release) - end -end diff --git a/app/workers/packages/nuget/extraction_worker.rb b/app/workers/packages/nuget/extraction_worker.rb new file mode 100644 index 00000000000..820304a9f3b --- /dev/null +++ b/app/workers/packages/nuget/extraction_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class ExtractionWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + queue_namespace :package_repositories + feature_category :package_registry + + def perform(package_file_id) + package_file = ::Packages::PackageFile.find_by_id(package_file_id) + + return unless package_file + + ::Packages::Nuget::UpdatePackageFromMetadataService.new(package_file).execute + + rescue ::Packages::Nuget::MetadataExtractionService::ExtractionError, + ::Packages::Nuget::UpdatePackageFromMetadataService::InvalidMetadataError => e + Gitlab::ErrorTracking.log_exception(e, project_id: package_file.project_id) + package_file.package.destroy! + end + end + end +end diff --git a/app/workers/partition_creation_worker.rb b/app/workers/partition_creation_worker.rb new file mode 100644 index 00000000000..9101623d93a --- /dev/null +++ b/app/workers/partition_creation_worker.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class PartitionCreationWorker + include ApplicationWorker + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + feature_category :database + idempotent! + + def perform + Gitlab::AppLogger.info("Checking state of dynamic postgres partitions") + + Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions + end +end diff --git a/app/workers/pipeline_update_worker.rb b/app/workers/pipeline_update_worker.rb index 7f667057af6..267caa5bedd 100644 --- a/app/workers/pipeline_update_worker.rb +++ b/app/workers/pipeline_update_worker.rb @@ -1,12 +1,14 @@ # frozen_string_literal: true -class PipelineUpdateWorker # rubocop:disable Scalability/IdempotentWorker +class PipelineUpdateWorker include ApplicationWorker include PipelineQueue queue_namespace :pipeline_processing urgency :high + idempotent! + def perform(pipeline_id) Ci::Pipeline.find_by_id(pipeline_id)&.update_legacy_status end diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index 62d76294bc0..8f844bd0b47 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -79,7 +79,7 @@ class PostReceive # rubocop:disable Scalability/IdempotentWorker return false unless user expire_caches(post_received, snippet.repository) - snippet.repository.expire_statistics_caches + Snippets::UpdateStatisticsService.new(snippet).execute end # Expire the repository status, branch, and tag cache once per push. diff --git a/app/workers/process_commit_worker.rb b/app/workers/process_commit_worker.rb index 5756ebb8358..3c7af641f16 100644 --- a/app/workers/process_commit_worker.rb +++ b/app/workers/process_commit_worker.rb @@ -80,7 +80,7 @@ class ProcessCommitWorker # manually parse these values. hash.each do |key, value| if key.to_s.end_with?(date_suffix) && value.is_a?(String) - hash[key] = Time.parse(value) + hash[key] = Time.zone.parse(value) end end diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index d29348e85bc..6c8640138a1 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -3,12 +3,13 @@ class ProjectExportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExceptionBacktrace - include ProjectExportOptions feature_category :importers worker_resource_boundary :memory urgency :throttled loggable_arguments 2, 3 + sidekiq_options retry: false + sidekiq_options status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION def perform(current_user_id, project_id, after_export_strategy = {}, params = {}) current_user = User.find(current_user_id) diff --git a/app/workers/project_update_repository_storage_worker.rb b/app/workers/project_update_repository_storage_worker.rb index 5c1a8062f12..7c0b1ae07fa 100644 --- a/app/workers/project_update_repository_storage_worker.rb +++ b/app/workers/project_update_repository_storage_worker.rb @@ -1,9 +1,11 @@ # frozen_string_literal: true -class ProjectUpdateRepositoryStorageWorker # rubocop:disable Scalability/IdempotentWorker +class ProjectUpdateRepositoryStorageWorker include ApplicationWorker + idempotent! feature_category :gitaly + urgency :throttled def perform(project_id, new_repository_storage_key, repository_storage_move_id = nil) repository_storage_move = diff --git a/app/workers/repository_check/batch_worker.rb b/app/workers/repository_check/batch_worker.rb index 1e2cb912598..d47f738ccb0 100644 --- a/app/workers/repository_check/batch_worker.rb +++ b/app/workers/repository_check/batch_worker.rb @@ -34,7 +34,7 @@ module RepositoryCheck end def perform_repository_checks - start = Time.now + start = Time.current # This loop will break after a little more than one hour ('a little # more' because `git fsck` may take a few minutes), or if it runs out of @@ -42,7 +42,7 @@ module RepositoryCheck # RepositoryCheckWorker each hour so that as long as there are repositories to # check, only one (or two) will be checked at a time. project_ids.each do |project_id| - break if Time.now - start >= RUN_TIME + break if Time.current - start >= RUN_TIME next unless try_obtain_lease_for_project(project_id) diff --git a/app/workers/repository_check/single_repository_worker.rb b/app/workers/repository_check/single_repository_worker.rb index edff7fc31df..d757b87c23a 100644 --- a/app/workers/repository_check/single_repository_worker.rb +++ b/app/workers/repository_check/single_repository_worker.rb @@ -17,7 +17,7 @@ module RepositoryCheck def update_repository_check_status(project, healthy) project.update_columns( last_repository_check_failed: !healthy, - last_repository_check_at: Time.now + last_repository_check_at: Time.current ) end diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 30570a2227e..54052bda675 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -4,10 +4,11 @@ class RepositoryImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExceptionBacktrace include ProjectStartImport - include ProjectImportOptions feature_category :importers worker_has_external_dependencies! + sidekiq_options retry: false + sidekiq_options status_expiration: Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION # technical debt: https://gitlab.com/gitlab-org/gitlab/issues/33991 sidekiq_options memory_killer_memory_growth_kb: ENV.fetch('MEMORY_KILLER_REPOSITORY_IMPORT_WORKER_MEMORY_GROWTH_KB', 50).to_i diff --git a/app/workers/service_desk_email_receiver_worker.rb b/app/workers/service_desk_email_receiver_worker.rb new file mode 100644 index 00000000000..8649034445c --- /dev/null +++ b/app/workers/service_desk_email_receiver_worker.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class ServiceDeskEmailReceiverWorker < EmailReceiverWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + def perform(raw) + return unless ::Gitlab::ServiceDeskEmail.enabled? + + begin + Gitlab::Email::ServiceDeskReceiver.new(raw).execute + rescue => e + handle_failure(raw, e) + end + end +end diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb deleted file mode 100644 index ce8d5bf0219..00000000000 --- a/app/workers/stuck_import_jobs_worker.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -class StuckImportJobsWorker # rubocop:disable Scalability/IdempotentWorker - include Gitlab::Import::StuckImportJob - - private - - def track_metrics(with_jid_count, without_jid_count) - Gitlab::Metrics.add_event( - :stuck_import_jobs, - projects_without_jid_count: without_jid_count, - projects_with_jid_count: with_jid_count - ) - end - - def enqueued_import_states - ProjectImportState.with_status([:scheduled, :started]) - end -end diff --git a/app/workers/update_container_registry_info_worker.rb b/app/workers/update_container_registry_info_worker.rb new file mode 100644 index 00000000000..14a816f25ef --- /dev/null +++ b/app/workers/update_container_registry_info_worker.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class UpdateContainerRegistryInfoWorker + include ApplicationWorker + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + feature_category :container_registry + urgency :low + + idempotent! + + def perform + UpdateContainerRegistryInfoService.new.execute + end +end |