summaryrefslogtreecommitdiff
path: root/app/workers
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2021-02-18 10:34:06 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2021-02-18 10:34:06 +0000
commit859a6fb938bb9ee2a317c46dfa4fcc1af49608f0 (patch)
treed7f2700abe6b4ffcb2dcfc80631b2d87d0609239 /app/workers
parent446d496a6d000c73a304be52587cd9bbc7493136 (diff)
downloadgitlab-ce-859a6fb938bb9ee2a317c46dfa4fcc1af49608f0.tar.gz
Add latest changes from gitlab-org/gitlab@13-9-stable-eev13.9.0-rc42
Diffstat (limited to 'app/workers')
-rw-r--r--app/workers/all_queues.yml71
-rw-r--r--app/workers/authorized_projects_worker.rb2
-rw-r--r--app/workers/build_coverage_worker.rb14
-rw-r--r--app/workers/build_finished_worker.rb6
-rw-r--r--app/workers/build_hooks_worker.rb3
-rw-r--r--app/workers/build_trace_sections_worker.rb14
-rw-r--r--app/workers/bulk_import_worker.rb4
-rw-r--r--app/workers/bulk_imports/entity_worker.rb10
-rw-r--r--app/workers/ci/build_report_result_worker.rb18
-rw-r--r--app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb20
-rw-r--r--app/workers/concerns/git_garbage_collect_methods.rb135
-rw-r--r--app/workers/container_expiration_policies/cleanup_container_repository_worker.rb5
-rw-r--r--app/workers/git_garbage_collect_worker.rb118
-rw-r--r--app/workers/issuable_export_csv_worker.rb25
-rw-r--r--app/workers/jira_connect/sync_builds_worker.rb1
-rw-r--r--app/workers/jira_connect/sync_deployments_worker.rb1
-rw-r--r--app/workers/jira_connect/sync_feature_flags_worker.rb1
-rw-r--r--app/workers/merge_request_cleanup_refs_worker.rb2
-rw-r--r--app/workers/namespaces/in_product_marketing_emails_worker.rb17
-rw-r--r--app/workers/packages/composer/cache_cleanup_worker.rb30
-rw-r--r--app/workers/pages_remove_worker.rb4
-rw-r--r--app/workers/pages_transfer_worker.rb2
-rw-r--r--app/workers/pipeline_hooks_worker.rb3
-rw-r--r--app/workers/post_receive.rb2
-rw-r--r--app/workers/projects/git_garbage_collect_worker.rb37
-rw-r--r--app/workers/schedule_merge_request_cleanup_refs_worker.rb1
-rw-r--r--app/workers/user_status_cleanup/batch_worker.rb33
-rw-r--r--app/workers/wikis/git_garbage_collect_worker.rb20
28 files changed, 391 insertions, 208 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 4c4a314a1e6..59ab0a4d05b 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -251,6 +251,14 @@
:weight: 1
:idempotent: true
:tags: []
+- :name: cronjob:namespaces_in_product_marketing_emails
+ :feature_category: :subgroups
+ :has_external_dependencies:
+ :urgency: :low
+ :resource_boundary: :unknown
+ :weight: 1
+ :idempotent:
+ :tags: []
- :name: cronjob:namespaces_prune_aggregation_schedules
:feature_category: :source_code_management
:has_external_dependencies:
@@ -259,6 +267,14 @@
:weight: 1
:idempotent:
:tags: []
+- :name: cronjob:packages_composer_cache_cleanup
+ :feature_category: :package_registry
+ :has_external_dependencies:
+ :urgency: :low
+ :resource_boundary: :unknown
+ :weight: 1
+ :idempotent: true
+ :tags: []
- :name: cronjob:pages_domain_removal_cron
:feature_category: :pages
:has_external_dependencies:
@@ -459,6 +475,14 @@
:weight: 1
:idempotent: true
:tags: []
+- :name: cronjob:user_status_cleanup_batch
+ :feature_category: :users
+ :has_external_dependencies:
+ :urgency: :low
+ :resource_boundary: :unknown
+ :weight: 1
+ :idempotent: true
+ :tags: []
- :name: cronjob:users_create_statistics
:feature_category: :users
:has_external_dependencies:
@@ -1076,24 +1100,23 @@
:idempotent:
:tags:
- :requires_disk_io
-- :name: pipeline_background:ci_build_report_result
+- :name: pipeline_background:ci_build_trace_chunk_flush
:feature_category: :continuous_integration
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
- :tags:
- - :requires_disk_io
-- :name: pipeline_background:ci_build_trace_chunk_flush
- :feature_category: :continuous_integration
+ :tags: []
+- :name: pipeline_background:ci_daily_build_group_report_results
+ :feature_category: :code_testing
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
:tags: []
-- :name: pipeline_background:ci_daily_build_group_report_results
+- :name: pipeline_background:ci_pipeline_artifacts_coverage_report
:feature_category: :code_testing
:has_external_dependencies:
:urgency: :low
@@ -1101,7 +1124,7 @@
:weight: 1
:idempotent: true
:tags: []
-- :name: pipeline_background:ci_pipeline_artifacts_coverage_report
+- :name: pipeline_background:ci_pipeline_artifacts_create_quality_report
:feature_category: :code_testing
:has_external_dependencies:
:urgency: :low
@@ -1165,24 +1188,6 @@
:weight: 4
:idempotent:
:tags: []
-- :name: pipeline_default:build_coverage
- :feature_category: :continuous_integration
- :has_external_dependencies:
- :urgency: :low
- :resource_boundary: :unknown
- :weight: 3
- :idempotent:
- :tags:
- - :requires_disk_io
-- :name: pipeline_default:build_trace_sections
- :feature_category: :continuous_integration
- :has_external_dependencies:
- :urgency: :low
- :resource_boundary: :unknown
- :weight: 3
- :idempotent:
- :tags:
- - :requires_disk_io
- :name: pipeline_default:ci_create_cross_project_pipeline
:feature_category: :continuous_integration
:has_external_dependencies:
@@ -1999,6 +2004,14 @@
:weight: 1
:idempotent: true
:tags: []
+- :name: projects_git_garbage_collect
+ :feature_category: :gitaly
+ :has_external_dependencies:
+ :urgency: :low
+ :resource_boundary: :unknown
+ :weight: 1
+ :idempotent:
+ :tags: []
- :name: prometheus_create_default_alerts
:feature_category: :incident_management
:has_external_dependencies:
@@ -2223,6 +2236,14 @@
:weight: 1
:idempotent: true
:tags: []
+- :name: wikis_git_garbage_collect
+ :feature_category: :gitaly
+ :has_external_dependencies:
+ :urgency: :low
+ :resource_boundary: :unknown
+ :weight: 1
+ :idempotent:
+ :tags: []
- :name: x509_certificate_revoke
:feature_category: :source_code_management
:has_external_dependencies:
diff --git a/app/workers/authorized_projects_worker.rb b/app/workers/authorized_projects_worker.rb
index f5132459131..6e07d6d0f71 100644
--- a/app/workers/authorized_projects_worker.rb
+++ b/app/workers/authorized_projects_worker.rb
@@ -25,7 +25,7 @@ class AuthorizedProjectsWorker
def perform(user_id)
user = User.find_by(id: user_id)
- user&.refresh_authorized_projects
+ user&.refresh_authorized_projects(source: self.class.name)
end
# rubocop: enable CodeReuse/ActiveRecord
end
diff --git a/app/workers/build_coverage_worker.rb b/app/workers/build_coverage_worker.rb
deleted file mode 100644
index d63d8549f09..00000000000
--- a/app/workers/build_coverage_worker.rb
+++ /dev/null
@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-class BuildCoverageWorker # rubocop:disable Scalability/IdempotentWorker
- include ApplicationWorker
- include PipelineQueue
-
- tags :requires_disk_io
-
- # rubocop: disable CodeReuse/ActiveRecord
- def perform(build_id)
- Ci::Build.find_by(id: build_id)&.update_coverage
- end
- # rubocop: enable CodeReuse/ActiveRecord
-end
diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb
index d7a5fcf4f18..4d15bcd16f7 100644
--- a/app/workers/build_finished_worker.rb
+++ b/app/workers/build_finished_worker.rb
@@ -29,9 +29,9 @@ class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker
# @param [Ci::Build] build The build to process.
def process_build(build)
# We execute these in sync to reduce IO.
- BuildTraceSectionsWorker.new.perform(build.id)
- BuildCoverageWorker.new.perform(build.id)
- Ci::BuildReportResultWorker.new.perform(build.id)
+ build.parse_trace_sections!
+ build.update_coverage
+ Ci::BuildReportResultService.new.execute(build)
# We execute these async as these are independent operations.
BuildHooksWorker.perform_async(build.id)
diff --git a/app/workers/build_hooks_worker.rb b/app/workers/build_hooks_worker.rb
index 9693d3eb57f..ce4aa7229aa 100644
--- a/app/workers/build_hooks_worker.rb
+++ b/app/workers/build_hooks_worker.rb
@@ -10,7 +10,8 @@ class BuildHooksWorker # rubocop:disable Scalability/IdempotentWorker
# rubocop: disable CodeReuse/ActiveRecord
def perform(build_id)
- Ci::Build.find_by(id: build_id)
+ Ci::Build.includes({ runner: :tags })
+ .find_by(id: build_id)
.try(:execute_hooks)
end
# rubocop: enable CodeReuse/ActiveRecord
diff --git a/app/workers/build_trace_sections_worker.rb b/app/workers/build_trace_sections_worker.rb
deleted file mode 100644
index 59f019b827e..00000000000
--- a/app/workers/build_trace_sections_worker.rb
+++ /dev/null
@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-class BuildTraceSectionsWorker # rubocop:disable Scalability/IdempotentWorker
- include ApplicationWorker
- include PipelineQueue
-
- tags :requires_disk_io
-
- # rubocop: disable CodeReuse/ActiveRecord
- def perform(build_id)
- Ci::Build.find_by(id: build_id)&.parse_trace_sections!
- end
- # rubocop: enable CodeReuse/ActiveRecord
-end
diff --git a/app/workers/bulk_import_worker.rb b/app/workers/bulk_import_worker.rb
index 81099d4e5f7..e6bc54895a7 100644
--- a/app/workers/bulk_import_worker.rb
+++ b/app/workers/bulk_import_worker.rb
@@ -27,6 +27,10 @@ class BulkImportWorker # rubocop:disable Scalability/IdempotentWorker
end
re_enqueue
+ rescue => e
+ Gitlab::ErrorTracking.track_exception(e, bulk_import_id: @bulk_import&.id)
+
+ @bulk_import&.fail_op
end
private
diff --git a/app/workers/bulk_imports/entity_worker.rb b/app/workers/bulk_imports/entity_worker.rb
index 9b29ad8f326..5b41ccbdea1 100644
--- a/app/workers/bulk_imports/entity_worker.rb
+++ b/app/workers/bulk_imports/entity_worker.rb
@@ -18,6 +18,16 @@ module BulkImports
BulkImports::Importers::GroupImporter.new(entity).execute
end
+
+ rescue => e
+ extra = {
+ bulk_import_id: entity&.bulk_import&.id,
+ entity_id: entity&.id
+ }
+
+ Gitlab::ErrorTracking.track_exception(e, extra)
+
+ entity&.fail_op
end
end
end
diff --git a/app/workers/ci/build_report_result_worker.rb b/app/workers/ci/build_report_result_worker.rb
deleted file mode 100644
index 01a45490541..00000000000
--- a/app/workers/ci/build_report_result_worker.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Ci
- class BuildReportResultWorker
- include ApplicationWorker
- include PipelineBackgroundQueue
-
- tags :requires_disk_io
-
- idempotent!
-
- def perform(build_id)
- Ci::Build.find_by_id(build_id).try do |build|
- Ci::BuildReportResultService.new.execute(build)
- end
- end
- end
-end
diff --git a/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb b/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb
new file mode 100644
index 00000000000..810106e8d9c
--- /dev/null
+++ b/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Ci
+ module PipelineArtifacts
+ class CreateQualityReportWorker
+ include ApplicationWorker
+
+ queue_namespace :pipeline_background
+ feature_category :code_testing
+
+ idempotent!
+
+ def perform(pipeline_id)
+ Ci::Pipeline.find_by_id(pipeline_id).try do |pipeline|
+ Ci::PipelineArtifacts::CreateCodeQualityMrDiffReportService.new.execute(pipeline)
+ end
+ end
+ end
+ end
+end
diff --git a/app/workers/concerns/git_garbage_collect_methods.rb b/app/workers/concerns/git_garbage_collect_methods.rb
new file mode 100644
index 00000000000..17a80d1ddb3
--- /dev/null
+++ b/app/workers/concerns/git_garbage_collect_methods.rb
@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module GitGarbageCollectMethods
+ extend ActiveSupport::Concern
+
+ included do
+ include ApplicationWorker
+
+ sidekiq_options retry: false
+ feature_category :gitaly
+ loggable_arguments 1, 2, 3
+ end
+
+ # Timeout set to 24h
+ LEASE_TIMEOUT = 86400
+
+ def perform(resource_id, task = :gc, lease_key = nil, lease_uuid = nil)
+ resource = find_resource(resource_id)
+ lease_key ||= default_lease_key(task, resource)
+ active_uuid = get_lease_uuid(lease_key)
+
+ if active_uuid
+ return unless active_uuid == lease_uuid
+
+ renew_lease(lease_key, active_uuid)
+ else
+ lease_uuid = try_obtain_lease(lease_key)
+
+ return unless lease_uuid
+ end
+
+ task = task.to_sym
+
+ before_gitaly_call(task, resource)
+ gitaly_call(task, resource)
+
+ # Refresh the branch cache in case garbage collection caused a ref lookup to fail
+ flush_ref_caches(resource) if gc?(task)
+
+ update_repository_statistics(resource) if task != :pack_refs
+
+ # In case pack files are deleted, release libgit2 cache and open file
+ # descriptors ASAP instead of waiting for Ruby garbage collection
+ resource.cleanup
+ ensure
+ cancel_lease(lease_key, lease_uuid) if lease_key.present? && lease_uuid.present?
+ end
+
+ private
+
+ def default_lease_key(task, resource)
+ "git_gc:#{task}:#{resource.class.name.underscore.pluralize}:#{resource.id}"
+ end
+
+ def find_resource(id)
+ raise NotImplementedError
+ end
+
+ def gc?(task)
+ task == :gc || task == :prune
+ end
+
+ def try_obtain_lease(key)
+ ::Gitlab::ExclusiveLease.new(key, timeout: LEASE_TIMEOUT).try_obtain
+ end
+
+ def renew_lease(key, uuid)
+ ::Gitlab::ExclusiveLease.new(key, uuid: uuid, timeout: LEASE_TIMEOUT).renew
+ end
+
+ def cancel_lease(key, uuid)
+ ::Gitlab::ExclusiveLease.cancel(key, uuid)
+ end
+
+ def get_lease_uuid(key)
+ ::Gitlab::ExclusiveLease.get_uuid(key)
+ end
+
+ def before_gitaly_call(task, resource)
+ # no-op
+ end
+
+ def gitaly_call(task, resource)
+ repository = resource.repository.raw_repository
+
+ client = get_gitaly_client(task, repository)
+
+ case task
+ when :prune, :gc
+ client.garbage_collect(bitmaps_enabled?, prune: task == :prune)
+ when :full_repack
+ client.repack_full(bitmaps_enabled?)
+ when :incremental_repack
+ client.repack_incremental
+ when :pack_refs
+ client.pack_refs
+ end
+ rescue GRPC::NotFound => e
+ Gitlab::GitLogger.error("#{__method__} failed:\nRepository not found")
+ raise Gitlab::Git::Repository::NoRepository.new(e)
+ rescue GRPC::BadStatus => e
+ Gitlab::GitLogger.error("#{__method__} failed:\n#{e}")
+ raise Gitlab::Git::CommandError.new(e)
+ end
+
+ def get_gitaly_client(task, repository)
+ if task == :pack_refs
+ Gitlab::GitalyClient::RefService
+ else
+ Gitlab::GitalyClient::RepositoryService
+ end.new(repository)
+ end
+
+ def bitmaps_enabled?
+ Gitlab::CurrentSettings.housekeeping_bitmaps_enabled
+ end
+
+ def flush_ref_caches(resource)
+ resource.repository.expire_branches_cache
+ resource.repository.branch_names
+ resource.repository.has_visible_content?
+ end
+
+ def update_repository_statistics(resource)
+ resource.repository.expire_statistics_caches
+
+ return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary
+
+ update_db_repository_statistics(resource)
+ end
+
+ def update_db_repository_statistics(resource)
+ # no-op
+ end
+end
diff --git a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb
index 7c86b194574..53220a7afed 100644
--- a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb
+++ b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb
@@ -18,6 +18,7 @@ module ContainerExpirationPolicies
cleanup_tags_service_before_truncate_size
cleanup_tags_service_after_truncate_size
cleanup_tags_service_before_delete_size
+ cleanup_tags_service_deleted_size
].freeze
def perform_work
@@ -25,6 +26,7 @@ module ContainerExpirationPolicies
return unless container_repository
log_extra_metadata_on_done(:container_repository_id, container_repository.id)
+ log_extra_metadata_on_done(:project_id, project.id)
unless allowed_to_run?(container_repository)
container_repository.cleanup_unscheduled!
@@ -78,7 +80,7 @@ module ContainerExpirationPolicies
end
def project
- container_repository&.project
+ container_repository.project
end
def container_repository
@@ -116,6 +118,7 @@ module ContainerExpirationPolicies
after_truncate_size &&
before_truncate_size != after_truncate_size
log_extra_metadata_on_done(:cleanup_tags_service_truncated, !!truncated)
+ log_extra_metadata_on_done(:running_jobs_count, running_jobs_count)
end
end
end
diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb
index e1dcb16bafb..a2aab23db7b 100644
--- a/app/workers/git_garbage_collect_worker.rb
+++ b/app/workers/git_garbage_collect_worker.rb
@@ -1,5 +1,11 @@
# frozen_string_literal: true
+# According to our docs, we can only remove workers on major releases
+# https://docs.gitlab.com/ee/development/sidekiq_style_guide.html#removing-workers.
+#
+# We need to still maintain this until 14.0 but with the current functionality.
+#
+# In https://gitlab.com/gitlab-org/gitlab/-/issues/299290 we track that removal.
class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
@@ -7,117 +13,7 @@ class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker
feature_category :gitaly
loggable_arguments 1, 2, 3
- # Timeout set to 24h
- LEASE_TIMEOUT = 86400
-
def perform(project_id, task = :gc, lease_key = nil, lease_uuid = nil)
- lease_key ||= "git_gc:#{task}:#{project_id}"
- project = Project.find(project_id)
- active_uuid = get_lease_uuid(lease_key)
-
- if active_uuid
- return unless active_uuid == lease_uuid
-
- renew_lease(lease_key, active_uuid)
- else
- lease_uuid = try_obtain_lease(lease_key)
-
- return unless lease_uuid
- end
-
- task = task.to_sym
-
- if gc?(task)
- ::Projects::GitDeduplicationService.new(project).execute
- cleanup_orphan_lfs_file_references(project)
- end
-
- gitaly_call(task, project)
-
- # Refresh the branch cache in case garbage collection caused a ref lookup to fail
- flush_ref_caches(project) if gc?(task)
-
- update_repository_statistics(project) if task != :pack_refs
-
- # In case pack files are deleted, release libgit2 cache and open file
- # descriptors ASAP instead of waiting for Ruby garbage collection
- project.cleanup
- ensure
- cancel_lease(lease_key, lease_uuid) if lease_key.present? && lease_uuid.present?
- end
-
- private
-
- def gc?(task)
- task == :gc || task == :prune
- end
-
- def try_obtain_lease(key)
- ::Gitlab::ExclusiveLease.new(key, timeout: LEASE_TIMEOUT).try_obtain
- end
-
- def renew_lease(key, uuid)
- ::Gitlab::ExclusiveLease.new(key, uuid: uuid, timeout: LEASE_TIMEOUT).renew
- end
-
- def cancel_lease(key, uuid)
- ::Gitlab::ExclusiveLease.cancel(key, uuid)
- end
-
- def get_lease_uuid(key)
- ::Gitlab::ExclusiveLease.get_uuid(key)
- end
-
- def gitaly_call(task, project)
- repository = project.repository.raw_repository
-
- client = if task == :pack_refs
- Gitlab::GitalyClient::RefService.new(repository)
- else
- Gitlab::GitalyClient::RepositoryService.new(repository)
- end
-
- case task
- when :prune, :gc
- client.garbage_collect(bitmaps_enabled?, prune: task == :prune)
- when :full_repack
- client.repack_full(bitmaps_enabled?)
- when :incremental_repack
- client.repack_incremental
- when :pack_refs
- client.pack_refs
- end
- rescue GRPC::NotFound => e
- Gitlab::GitLogger.error("#{__method__} failed:\nRepository not found")
- raise Gitlab::Git::Repository::NoRepository.new(e)
- rescue GRPC::BadStatus => e
- Gitlab::GitLogger.error("#{__method__} failed:\n#{e}")
- raise Gitlab::Git::CommandError.new(e)
- end
-
- def cleanup_orphan_lfs_file_references(project)
- return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary
-
- ::Gitlab::Cleanup::OrphanLfsFileReferences.new(project, dry_run: false, logger: logger).run!
- rescue => err
- Gitlab::GitLogger.warn(message: "Cleaning up orphan LFS objects files failed", error: err.message)
- Gitlab::ErrorTracking.track_and_raise_for_dev_exception(err)
- end
-
- def flush_ref_caches(project)
- project.repository.expire_branches_cache
- project.repository.branch_names
- project.repository.has_visible_content?
- end
-
- def update_repository_statistics(project)
- project.repository.expire_statistics_caches
- return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary
-
- Projects::UpdateStatisticsService.new(project, nil, statistics: [:repository_size, :lfs_objects_size]).execute
- end
-
- def bitmaps_enabled?
- Gitlab::CurrentSettings.housekeeping_bitmaps_enabled
+ ::Projects::GitGarbageCollectWorker.new.perform(project_id, task, lease_key, lease_uuid)
end
end
diff --git a/app/workers/issuable_export_csv_worker.rb b/app/workers/issuable_export_csv_worker.rb
index 33452b14edb..eb96a78497c 100644
--- a/app/workers/issuable_export_csv_worker.rb
+++ b/app/workers/issuable_export_csv_worker.rb
@@ -10,29 +10,21 @@ class IssuableExportCsvWorker # rubocop:disable Scalability/IdempotentWorker
def perform(type, current_user_id, project_id, params)
user = User.find(current_user_id)
project = Project.find(project_id)
- finder_params = map_params(params, project_id)
- export_service(type.to_sym, user, project, finder_params).email(user)
+ export_service(type, user, project, params).email(user)
rescue ActiveRecord::RecordNotFound => error
logger.error("Failed to export CSV (current_user_id:#{current_user_id}, project_id:#{project_id}): #{error.message}")
end
private
- def map_params(params, project_id)
- params
- .symbolize_keys
- .except(:sort)
- .merge(project_id: project_id)
- end
-
def export_service(type, user, project, params)
- issuable_class = service_classes_for(type)
- issuables = issuable_class[:finder].new(user, params).execute
- issuable_class[:service].new(issuables, project)
+ issuable_classes = issuable_classes_for(type.to_sym)
+ issuables = issuable_classes[:finder].new(user, parse_params(params, project.id)).execute
+ issuable_classes[:service].new(issuables, project)
end
- def service_classes_for(type)
+ def issuable_classes_for(type)
case type
when :issue
{ finder: IssuesFinder, service: Issues::ExportCsvService }
@@ -43,6 +35,13 @@ class IssuableExportCsvWorker # rubocop:disable Scalability/IdempotentWorker
end
end
+ def parse_params(params, project_id)
+ params
+ .symbolize_keys
+ .except(:sort)
+ .merge(project_id: project_id)
+ end
+
def type_error_message(type)
"Type parameter must be :issue or :merge_request, it was #{type}"
end
diff --git a/app/workers/jira_connect/sync_builds_worker.rb b/app/workers/jira_connect/sync_builds_worker.rb
index c1c749f6041..9cb5d5d247d 100644
--- a/app/workers/jira_connect/sync_builds_worker.rb
+++ b/app/workers/jira_connect/sync_builds_worker.rb
@@ -14,7 +14,6 @@ module JiraConnect
pipeline = Ci::Pipeline.find_by_id(pipeline_id)
return unless pipeline
- return unless Feature.enabled?(:jira_sync_builds, pipeline.project)
::JiraConnect::SyncService
.new(pipeline.project)
diff --git a/app/workers/jira_connect/sync_deployments_worker.rb b/app/workers/jira_connect/sync_deployments_worker.rb
index 0f261e29464..7272d35f4cb 100644
--- a/app/workers/jira_connect/sync_deployments_worker.rb
+++ b/app/workers/jira_connect/sync_deployments_worker.rb
@@ -14,7 +14,6 @@ module JiraConnect
deployment = Deployment.find_by_id(deployment_id)
return unless deployment
- return unless Feature.enabled?(:jira_sync_deployments, deployment.project)
::JiraConnect::SyncService
.new(deployment.project)
diff --git a/app/workers/jira_connect/sync_feature_flags_worker.rb b/app/workers/jira_connect/sync_feature_flags_worker.rb
index 7e98d0eada7..496b9f1626d 100644
--- a/app/workers/jira_connect/sync_feature_flags_worker.rb
+++ b/app/workers/jira_connect/sync_feature_flags_worker.rb
@@ -14,7 +14,6 @@ module JiraConnect
feature_flag = ::Operations::FeatureFlag.find_by_id(feature_flag_id)
return unless feature_flag
- return unless Feature.enabled?(:jira_sync_feature_flags, feature_flag.project)
::JiraConnect::SyncService
.new(feature_flag.project)
diff --git a/app/workers/merge_request_cleanup_refs_worker.rb b/app/workers/merge_request_cleanup_refs_worker.rb
index 6b991a2253f..fbd62ac0a91 100644
--- a/app/workers/merge_request_cleanup_refs_worker.rb
+++ b/app/workers/merge_request_cleanup_refs_worker.rb
@@ -7,6 +7,8 @@ class MergeRequestCleanupRefsWorker
idempotent!
def perform(merge_request_id)
+ return unless Feature.enabled?(:merge_request_refs_cleanup, default_enabled: false)
+
merge_request = MergeRequest.find_by_id(merge_request_id)
unless merge_request
diff --git a/app/workers/namespaces/in_product_marketing_emails_worker.rb b/app/workers/namespaces/in_product_marketing_emails_worker.rb
new file mode 100644
index 00000000000..66d140928a7
--- /dev/null
+++ b/app/workers/namespaces/in_product_marketing_emails_worker.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Namespaces
+ class InProductMarketingEmailsWorker # rubocop:disable Scalability/IdempotentWorker
+ include ApplicationWorker
+ include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
+
+ feature_category :subgroups
+ urgency :low
+
+ def perform
+ return unless Gitlab::Experimentation.active?(:in_product_marketing_emails)
+
+ Namespaces::InProductMarketingEmailsService.send_for_all_tracks_and_intervals
+ end
+ end
+end
diff --git a/app/workers/packages/composer/cache_cleanup_worker.rb b/app/workers/packages/composer/cache_cleanup_worker.rb
new file mode 100644
index 00000000000..638e50e18c4
--- /dev/null
+++ b/app/workers/packages/composer/cache_cleanup_worker.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Packages
+ module Composer
+ class CacheCleanupWorker
+ include ApplicationWorker
+ include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
+
+ feature_category :package_registry
+
+ idempotent!
+
+ def perform
+ ::Packages::Composer::CacheFile.without_namespace.find_in_batches do |cache_files|
+ cache_files.each(&:destroy)
+ rescue ActiveRecord::RecordNotFound
+ # ignore. likely due to object already being deleted.
+ end
+
+ ::Packages::Composer::CacheFile.expired.find_in_batches do |cache_files|
+ cache_files.each(&:destroy)
+ rescue ActiveRecord::RecordNotFound
+ # ignore. likely due to object already being deleted.
+ end
+ rescue => e
+ Gitlab::ErrorTracking.log_exception(e)
+ end
+ end
+ end
+end
diff --git a/app/workers/pages_remove_worker.rb b/app/workers/pages_remove_worker.rb
index b83168fd7bd..67ea18545a7 100644
--- a/app/workers/pages_remove_worker.rb
+++ b/app/workers/pages_remove_worker.rb
@@ -1,5 +1,6 @@
# frozen_string_literal: true
+# TODO: remove this worker https://gitlab.com/gitlab-org/gitlab/-/issues/320775
class PagesRemoveWorker # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
@@ -11,7 +12,6 @@ class PagesRemoveWorker # rubocop:disable Scalability/IdempotentWorker
project = Project.find_by_id(project_id)
return unless project
- project.remove_pages
- project.pages_domains.delete_all
+ project.legacy_remove_pages
end
end
diff --git a/app/workers/pages_transfer_worker.rb b/app/workers/pages_transfer_worker.rb
index f78564cc69d..5d395c9e38a 100644
--- a/app/workers/pages_transfer_worker.rb
+++ b/app/workers/pages_transfer_worker.rb
@@ -9,7 +9,7 @@ class PagesTransferWorker # rubocop:disable Scalability/IdempotentWorker
loggable_arguments 0, 1
def perform(method, args)
- return unless Gitlab::PagesTransfer::Async::METHODS.include?(method)
+ return unless Gitlab::PagesTransfer::METHODS.include?(method)
result = Gitlab::PagesTransfer.new.public_send(method, *args) # rubocop:disable GitlabSecurity/PublicSend
diff --git a/app/workers/pipeline_hooks_worker.rb b/app/workers/pipeline_hooks_worker.rb
index 85ecdd02fb5..b8dd4768cfb 100644
--- a/app/workers/pipeline_hooks_worker.rb
+++ b/app/workers/pipeline_hooks_worker.rb
@@ -10,7 +10,8 @@ class PipelineHooksWorker # rubocop:disable Scalability/IdempotentWorker
# rubocop: disable CodeReuse/ActiveRecord
def perform(pipeline_id)
- Ci::Pipeline.find_by(id: pipeline_id)
+ Ci::Pipeline.includes({ builds: { runner: :tags } })
+ .find_by(id: pipeline_id)
.try(:execute_hooks)
end
# rubocop: enable CodeReuse/ActiveRecord
diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb
index 9fe7dd31e68..ac55f883fc5 100644
--- a/app/workers/post_receive.rb
+++ b/app/workers/post_receive.rb
@@ -2,6 +2,7 @@
class PostReceive # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
+ include Gitlab::Experiment::Dsl
feature_category :source_code_management
urgency :high
@@ -121,6 +122,7 @@ class PostReceive # rubocop:disable Scalability/IdempotentWorker
end
def after_project_changes_hooks(project, user, refs, changes)
+ experiment(:new_project_readme, actor: user).track_initial_writes(project)
repository_update_hook_data = Gitlab::DataBuilder::Repository.update(project, user, changes, refs)
SystemHooksService.new.execute_hooks(repository_update_hook_data, :repository_update_hooks)
Gitlab::UsageDataCounters::SourceCodeCounter.count(:pushes)
diff --git a/app/workers/projects/git_garbage_collect_worker.rb b/app/workers/projects/git_garbage_collect_worker.rb
new file mode 100644
index 00000000000..4f908529b34
--- /dev/null
+++ b/app/workers/projects/git_garbage_collect_worker.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Projects
+ class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker
+ extend ::Gitlab::Utils::Override
+ include GitGarbageCollectMethods
+
+ private
+
+ override :find_resource
+ def find_resource(id)
+ Project.find(id)
+ end
+
+ override :before_gitaly_call
+ def before_gitaly_call(task, resource)
+ return unless gc?(task)
+
+ ::Projects::GitDeduplicationService.new(resource).execute
+ cleanup_orphan_lfs_file_references(resource)
+ end
+
+ def cleanup_orphan_lfs_file_references(resource)
+ return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary
+
+ ::Gitlab::Cleanup::OrphanLfsFileReferences.new(resource, dry_run: false, logger: logger).run!
+ rescue => err
+ Gitlab::GitLogger.warn(message: "Cleaning up orphan LFS objects files failed", error: err.message)
+ Gitlab::ErrorTracking.track_and_raise_for_dev_exception(err)
+ end
+
+ override :update_db_repository_statistics
+ def update_db_repository_statistics(resource)
+ Projects::UpdateStatisticsService.new(resource, nil, statistics: [:repository_size, :lfs_objects_size]).execute
+ end
+ end
+end
diff --git a/app/workers/schedule_merge_request_cleanup_refs_worker.rb b/app/workers/schedule_merge_request_cleanup_refs_worker.rb
index 59b8993f78f..967032f99e5 100644
--- a/app/workers/schedule_merge_request_cleanup_refs_worker.rb
+++ b/app/workers/schedule_merge_request_cleanup_refs_worker.rb
@@ -16,6 +16,7 @@ class ScheduleMergeRequestCleanupRefsWorker
def perform
return if Gitlab::Database.read_only?
+ return unless Feature.enabled?(:merge_request_refs_cleanup, default_enabled: false)
ids = MergeRequest::CleanupSchedule.scheduled_merge_request_ids(LIMIT).map { |id| [id] }
diff --git a/app/workers/user_status_cleanup/batch_worker.rb b/app/workers/user_status_cleanup/batch_worker.rb
new file mode 100644
index 00000000000..0c1087cc4d2
--- /dev/null
+++ b/app/workers/user_status_cleanup/batch_worker.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module UserStatusCleanup
+ # This worker will run every minute to look for user status records to clean up.
+ class BatchWorker
+ include ApplicationWorker
+ # rubocop:disable Scalability/CronWorkerContext
+ include CronjobQueue
+ # rubocop:enable Scalability/CronWorkerContext
+
+ feature_category :users
+
+ idempotent!
+
+ # Avoid running too many UPDATE queries at once
+ MAX_RUNTIME = 30.seconds
+
+ def perform
+ return unless UserStatus.scheduled_for_cleanup.exists?
+
+ start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+ loop do
+ result = Users::BatchStatusCleanerService.execute
+ break if result[:deleted_rows] < Users::BatchStatusCleanerService::BATCH_SIZE
+
+ current_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+ break if (current_time - start_time) > MAX_RUNTIME
+ end
+ end
+ end
+end
diff --git a/app/workers/wikis/git_garbage_collect_worker.rb b/app/workers/wikis/git_garbage_collect_worker.rb
new file mode 100644
index 00000000000..1b455c50618
--- /dev/null
+++ b/app/workers/wikis/git_garbage_collect_worker.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Wikis
+ class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker
+ extend ::Gitlab::Utils::Override
+ include GitGarbageCollectMethods
+
+ private
+
+ override :find_resource
+ def find_resource(id)
+ Project.find(id).wiki
+ end
+
+ override :update_db_repository_statistics
+ def update_db_repository_statistics(resource)
+ Projects::UpdateStatisticsService.new(resource.container, nil, statistics: [:wiki_size]).execute
+ end
+ end
+end