diff options
Diffstat (limited to 'lib/gitlab/background_migration')
44 files changed, 302 insertions, 91 deletions
diff --git a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb new file mode 100644 index 00000000000..79e7a2f2279 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to set namespaces.traversal_ids in sub-batches, of all namespaces with + # a parent and not already set. + # rubocop:disable Style/Documentation + class BackfillNamespaceTraversalIdsChildren + class Namespace < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'namespaces' + + scope :base_query, -> { where.not(parent_id: nil) } + end + + PAUSE_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + batch_query = Namespace.base_query.where(id: start_id..end_id) + batch_query.each_batch(of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + ranged_query = Namespace.unscoped.base_query.where(id: first..last) + + update_sql = <<~SQL + UPDATE namespaces + SET traversal_ids = calculated_ids.traversal_ids + FROM #{calculated_traversal_ids(ranged_query)} calculated_ids + WHERE namespaces.id = calculated_ids.id + AND namespaces.traversal_ids = '{}' + SQL + ActiveRecord::Base.connection.execute(update_sql) + + sleep PAUSE_SECONDS + end + + # We have to add all arguments when marking a job as succeeded as they + # are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals` + mark_job_as_succeeded(start_id, end_id, sub_batch_size) + end + + private + + # Calculate the ancestor path for a given set of namespaces. + def calculated_traversal_ids(batch) + <<~SQL + ( + WITH RECURSIVE cte(source_id, namespace_id, parent_id, height) AS ( + ( + SELECT batch.id, batch.id, batch.parent_id, 1 + FROM (#{batch.to_sql}) AS batch + ) + UNION ALL + ( + SELECT cte.source_id, n.id, n.parent_id, cte.height+1 + FROM namespaces n, cte + WHERE n.id = cte.parent_id + ) + ) + SELECT flat_hierarchy.source_id as id, + array_agg(flat_hierarchy.namespace_id ORDER BY flat_hierarchy.height DESC) as traversal_ids + FROM (SELECT * FROM cte FOR UPDATE) flat_hierarchy + GROUP BY flat_hierarchy.source_id + ) + SQL + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'BackfillNamespaceTraversalIdsChildren', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb new file mode 100644 index 00000000000..1c0a83285a6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to set namespaces.traversal_ids in sub-batches, of all namespaces + # without a parent and not already set. + # rubocop:disable Style/Documentation + class BackfillNamespaceTraversalIdsRoots + class Namespace < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'namespaces' + + scope :base_query, -> { where(parent_id: nil) } + end + + PAUSE_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + ranged_query = Namespace.base_query + .where(id: start_id..end_id) + .where("traversal_ids = '{}'") + + ranged_query.each_batch(of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + + # The query need to be reconstructed because .each_batch modifies the default scope + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 + Namespace.unscoped + .base_query + .where(id: first..last) + .where("traversal_ids = '{}'") + .update_all('traversal_ids = ARRAY[id]') + + sleep PAUSE_SECONDS + end + + mark_job_as_succeeded(start_id, end_id, sub_batch_size) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'BackfillNamespaceTraversalIdsRoots', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb index 8befade8c3a..6f37f1846d2 100644 --- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb +++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb @@ -36,7 +36,7 @@ module Gitlab create_repository_and_files(snippet) logger.info(message: 'Snippet Migration: repository created and migrated', snippet: snippet.id) - rescue => e + rescue StandardError => e set_file_path_error(e) set_signature_error(e) @@ -68,7 +68,7 @@ module Gitlab # Removing the db record def destroy_snippet_repository(snippet) snippet.snippet_repository&.delete - rescue => e + rescue StandardError => e logger.error(message: "Snippet Migration: error destroying snippet repository. Reason: #{e.message}", snippet: snippet.id) end @@ -78,7 +78,7 @@ module Gitlab snippet.repository.remove snippet.repository.expire_exists_cache - rescue => e + rescue StandardError => e logger.error(message: "Snippet Migration: error deleting repository. Reason: #{e.message}", snippet: snippet.id) end diff --git a/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb b/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb index 83d60d2db19..41f7f7f2f24 100644 --- a/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb +++ b/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly.prepend_if_ee('EE::Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly') +Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly.prepend_mod_with('Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly') diff --git a/lib/gitlab/background_migration/calculate_wiki_sizes.rb b/lib/gitlab/background_migration/calculate_wiki_sizes.rb index 76598f6e2a6..7b334b9c1d0 100644 --- a/lib/gitlab/background_migration/calculate_wiki_sizes.rb +++ b/lib/gitlab/background_migration/calculate_wiki_sizes.rb @@ -9,7 +9,7 @@ module Gitlab .where(id: start_id..stop_id) .includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics| statistics.refresh!(only: [:wiki_size]) - rescue => e + rescue StandardError => e Gitlab::AppLogger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" end end diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index b89ea7dc250..529b8cdf8d4 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -2,8 +2,8 @@ module Gitlab module BackgroundMigration - # Background migration that updates the value of a - # column using the value of another column in the same table. + # Background migration that updates the value of one or more + # columns using the value of other columns in the same table. # # - The {start_id, end_id} arguments are at the start so that it can be used # with `queue_batched_background_migration` @@ -16,8 +16,6 @@ module Gitlab class CopyColumnUsingBackgroundMigrationJob include Gitlab::Database::DynamicModelHelpers - PAUSE_SECONDS = 0.1 - # start_id - The start ID of the range of rows to update. # end_id - The end ID of the range of rows to update. # batch_table - The name of the table that contains the columns. @@ -25,20 +23,26 @@ module Gitlab # sub_batch_size - We don't want updates to take more than ~100ms # This allows us to run multiple smaller batches during # the minimum 2.minute interval that we can schedule jobs - # copy_from - The column containing the data to copy. - # copy_to - The column to copy the data to. - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, copy_from, copy_to) - quoted_copy_from = connection.quote_column_name(copy_from) - quoted_copy_to = connection.quote_column_name(copy_to) + # pause_ms - The number of milliseconds to sleep between each subbatch execution. + # copy_from - List of columns containing the data to copy. + # copy_to - List of columns to copy the data to. Order must match the order in `copy_from`. + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, copy_from, copy_to) + copy_from = Array.wrap(copy_from) + copy_to = Array.wrap(copy_to) + + raise ArgumentError, 'number of source and destination columns must match' unless copy_from.count == copy_to.count + + assignment_clauses = column_assignment_clauses(copy_from, copy_to) parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| batch_metrics.time_operation(:update_all) do - sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}") + sub_batch.update_all(assignment_clauses) end - sleep(PAUSE_SECONDS) + pause_ms = 0 if pause_ms < 0 + sleep(pause_ms * 0.001) end end @@ -55,6 +59,17 @@ module Gitlab def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) define_batchable_model(source_table).where(source_key_column => start_id..stop_id) end + + def column_assignment_clauses(copy_from, copy_to) + assignments = copy_from.zip(copy_to).map do |from_column, to_column| + from_column = connection.quote_column_name(from_column) + to_column = connection.quote_column_name(to_column) + + "#{to_column} = #{from_column}" + end + + assignments.join(', ') + end end end end diff --git a/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb b/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb new file mode 100644 index 00000000000..293530f6536 --- /dev/null +++ b/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# rubocop: disable Style/Documentation +class Gitlab::BackgroundMigration::DropInvalidVulnerabilities + # rubocop: disable Gitlab/NamespacedClass + class Vulnerability < ActiveRecord::Base + self.table_name = "vulnerabilities" + has_many :findings, class_name: 'VulnerabilitiesFinding', inverse_of: :vulnerability + end + + class VulnerabilitiesFinding < ActiveRecord::Base + self.table_name = "vulnerability_occurrences" + belongs_to :vulnerability, class_name: 'Vulnerability', inverse_of: :findings, foreign_key: 'vulnerability_id' + end + # rubocop: enable Gitlab/NamespacedClass + + # rubocop: disable CodeReuse/ActiveRecord + def perform(start_id, end_id) + Vulnerability + .where(id: start_id..end_id) + .left_joins(:findings) + .where(vulnerability_occurrences: { vulnerability_id: nil }) + .delete_all + + mark_job_as_succeeded(start_id, end_id) + end + # rubocop: enable CodeReuse/ActiveRecord + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'DropInvalidVulnerabilities', + arguments + ) + end +end diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb index c0099d44b5a..7b5c32e3d6d 100644 --- a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb +++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb @@ -24,7 +24,7 @@ module Gitlab certificate_valid_not_before: domain.x509&.not_before&.iso8601, certificate_valid_not_after: domain.x509&.not_after&.iso8601 ) - rescue => e + rescue StandardError => e Gitlab::AppLogger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" end end diff --git a/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb b/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb index d6ec56ae19e..c50bf430d92 100644 --- a/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb +++ b/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::FixOrphanPromotedIssues.prepend_if_ee('EE::Gitlab::BackgroundMigration::FixOrphanPromotedIssues') +Gitlab::BackgroundMigration::FixOrphanPromotedIssues.prepend_mod_with('Gitlab::BackgroundMigration::FixOrphanPromotedIssues') diff --git a/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb b/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb index 46921a070c3..47a68c61fcc 100644 --- a/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb +++ b/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents.prepend_if_ee('EE::Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents') +Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents.prepend_mod_with('Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents') diff --git a/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb b/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb index 85bcf8558f2..160e6d2fe8b 100644 --- a/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb +++ b/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::GenerateGitlabSubscriptions.prepend_if_ee('EE::Gitlab::BackgroundMigration::GenerateGitlabSubscriptions') +Gitlab::BackgroundMigration::GenerateGitlabSubscriptions.prepend_mod_with('Gitlab::BackgroundMigration::GenerateGitlabSubscriptions') diff --git a/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb b/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb index 27b984b4531..ba66721f65c 100644 --- a/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb +++ b/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb @@ -12,4 +12,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateApproverToApprovalRules.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRules') +Gitlab::BackgroundMigration::MigrateApproverToApprovalRules.prepend_mod_with('Gitlab::BackgroundMigration::MigrateApproverToApprovalRules') diff --git a/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb index 053b7363286..4899c50b9cf 100644 --- a/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb +++ b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress') +Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress.prepend_mod_with('Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress') diff --git a/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb index 130f97b09d7..2855566d7e8 100644 --- a/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb +++ b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch') +Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch.prepend_mod_with('Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch') diff --git a/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb b/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb index de2d9909961..d85f980d3f1 100644 --- a/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb +++ b/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups') +Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups.prepend_mod_with('Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups') diff --git a/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb new file mode 100644 index 00000000000..68bbd3cfebb --- /dev/null +++ b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to migrate the context of project taggings from `tags` to `topics` + class MigrateProjectTaggingsContextFromTagsToTopics + # Temporary AR table for taggings + class Tagging < ActiveRecord::Base + include EachBatch + + self.table_name = 'taggings' + end + + def perform(start_id, stop_id) + Tagging.where(taggable_type: 'Project', context: 'tags', id: start_id..stop_id).each_batch(of: 500) do |relation| + relation.update_all(context: 'topics') + end + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_security_scans.rb b/lib/gitlab/background_migration/migrate_security_scans.rb index 189a150cb87..0ae984f2dbc 100644 --- a/lib/gitlab/background_migration/migrate_security_scans.rb +++ b/lib/gitlab/background_migration/migrate_security_scans.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateSecurityScans.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateSecurityScans') +Gitlab::BackgroundMigration::MigrateSecurityScans.prepend_mod_with('Gitlab::BackgroundMigration::MigrateSecurityScans') diff --git a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb index 9ecf53317d0..c01545e5dca 100644 --- a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb +++ b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb @@ -8,18 +8,15 @@ module Gitlab class MoveContainerRegistryEnabledToProjectFeature MAX_BATCH_SIZE = 300 - module Migratable - # Migration model namespace isolated from application code. - class ProjectFeature < ActiveRecord::Base - ENABLED = 20 - DISABLED = 0 - end - end + ENABLED = 20 + DISABLED = 0 def perform(from_id, to_id) (from_id..to_id).each_slice(MAX_BATCH_SIZE) do |batch| process_batch(batch.first, batch.last) end + + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('MoveContainerRegistryEnabledToProjectFeature', [from_id, to_id]) end private @@ -37,9 +34,9 @@ module Gitlab <<~SQL UPDATE project_features SET container_registry_access_level = (CASE p.container_registry_enabled - WHEN true THEN #{ProjectFeature::ENABLED} - WHEN false THEN #{ProjectFeature::DISABLED} - ELSE #{ProjectFeature::DISABLED} + WHEN true THEN #{ENABLED} + WHEN false THEN #{DISABLED} + ELSE #{DISABLED} END) FROM projects p WHERE project_id = p.id AND diff --git a/lib/gitlab/background_migration/move_epic_issues_after_epics.rb b/lib/gitlab/background_migration/move_epic_issues_after_epics.rb index dc982e703d1..174994c7862 100644 --- a/lib/gitlab/background_migration/move_epic_issues_after_epics.rb +++ b/lib/gitlab/background_migration/move_epic_issues_after_epics.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics.prepend_if_ee('EE::Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics') +Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics.prepend_mod_with('Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics') diff --git a/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb b/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb index c3c0db2495c..890a43800c9 100644 --- a/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb +++ b/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb @@ -11,4 +11,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests') +Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests.prepend_mod_with('Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests') diff --git a/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb b/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb index 2243c7531c0..ac7ed18ba14 100644 --- a/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb +++ b/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb @@ -11,4 +11,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects') +Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects.prepend_mod_with('Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects') diff --git a/lib/gitlab/background_migration/populate_namespace_statistics.rb b/lib/gitlab/background_migration/populate_namespace_statistics.rb index e352ae71de6..e873ad412f2 100644 --- a/lib/gitlab/background_migration/populate_namespace_statistics.rb +++ b/lib/gitlab/background_migration/populate_namespace_statistics.rb @@ -13,4 +13,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateNamespaceStatistics.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateNamespaceStatistics') +Gitlab::BackgroundMigration::PopulateNamespaceStatistics.prepend_mod_with('Gitlab::BackgroundMigration::PopulateNamespaceStatistics') diff --git a/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb b/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb index e8f436b183e..ed7ffce8018 100644 --- a/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb +++ b/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb @@ -33,7 +33,7 @@ module Gitlab def update_namespace_statistics(namespace) Namespaces::StatisticsRefresherService.new.execute(namespace) - rescue => e + rescue StandardError => e error_message("Error updating statistics for namespace #{namespace.id}: #{e.message}") end diff --git a/lib/gitlab/background_migration/populate_project_snippet_statistics.rb b/lib/gitlab/background_migration/populate_project_snippet_statistics.rb index 7659b63271f..37af320f044 100644 --- a/lib/gitlab/background_migration/populate_project_snippet_statistics.rb +++ b/lib/gitlab/background_migration/populate_project_snippet_statistics.rb @@ -11,12 +11,12 @@ module Gitlab namespace_snippets.group_by(&:project).each do |project, snippets| upsert_snippet_statistics(snippets) update_project_statistics(project) - rescue + rescue StandardError error_message("Error updating statistics for project #{project.id}") end update_namespace_statistics(namespace_snippets.first.project.root_namespace) - rescue => e + rescue StandardError => e error_message("Error updating statistics for namespace #{namespace_id}: #{e.message}") end end diff --git a/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb b/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb index eb72ef1de33..e95955c450d 100644 --- a/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb +++ b/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb @@ -9,4 +9,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn') +Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn.prepend_mod_with('Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn') diff --git a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb index 4aff9d1e2c1..175966b940d 100644 --- a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb +++ b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb @@ -15,4 +15,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings.prepend_if_ee('::EE::Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings') +Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings.prepend_mod_with('Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings') diff --git a/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb index fc79f7125e3..8241fea66db 100644 --- a/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb +++ b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId') +Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId.prepend_mod_with('Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId') diff --git a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb index 2e81b1615d8..9a9f23e29ea 100644 --- a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb +++ b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb @@ -11,4 +11,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics') +Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics.prepend_mod_with('Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics') diff --git a/lib/gitlab/background_migration/prune_orphaned_geo_events.rb b/lib/gitlab/background_migration/prune_orphaned_geo_events.rb index 8b16db8be35..0efbe72775c 100644 --- a/lib/gitlab/background_migration/prune_orphaned_geo_events.rb +++ b/lib/gitlab/background_migration/prune_orphaned_geo_events.rb @@ -14,4 +14,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PruneOrphanedGeoEvents.prepend_if_ee('EE::Gitlab::BackgroundMigration::PruneOrphanedGeoEvents') +Gitlab::BackgroundMigration::PruneOrphanedGeoEvents.prepend_mod_with('Gitlab::BackgroundMigration::PruneOrphanedGeoEvents') diff --git a/lib/gitlab/background_migration/recalculate_project_authorizations.rb b/lib/gitlab/background_migration/recalculate_project_authorizations.rb index 3d2ce9fc10c..6a250a96c94 100644 --- a/lib/gitlab/background_migration/recalculate_project_authorizations.rb +++ b/lib/gitlab/background_migration/recalculate_project_authorizations.rb @@ -5,37 +5,7 @@ module Gitlab # rubocop:disable Style/Documentation class RecalculateProjectAuthorizations def perform(user_ids) - user_ids.each do |user_id| - user = User.find_by(id: user_id) - - next unless user - - service = Users::RefreshAuthorizedProjectsService.new( - user, - incorrect_auth_found_callback: - ->(project_id, access_level) do - logger.info(message: 'Removing ProjectAuthorizations', - user_id: user.id, - project_id: project_id, - access_level: access_level) - end, - missing_auth_found_callback: - ->(project_id, access_level) do - logger.info(message: 'Creating ProjectAuthorizations', - user_id: user.id, - project_id: project_id, - access_level: access_level) - end - ) - - service.execute - end - end - - private - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build + # no-op end end end diff --git a/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb b/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb index cc9b0329556..17ef6dec4c0 100644 --- a/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb +++ b/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveDuplicateCsFindings.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveDuplicateCsFindings') +Gitlab::BackgroundMigration::RemoveDuplicateCsFindings.prepend_mod_with('Gitlab::BackgroundMigration::RemoveDuplicateCsFindings') diff --git a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb index cd305adc7cd..e5772fc7375 100644 --- a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb +++ b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId') +Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId.prepend_mod_with('Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId') diff --git a/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb b/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb index 74c48b237cc..cb6a600a525 100644 --- a/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb +++ b/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos') +Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos.prepend_mod_with('Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos') diff --git a/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb b/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb index 3920e8dc2de..540ffc6f548 100644 --- a/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb +++ b/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel') +Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel') diff --git a/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb b/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb index f137e41c728..cecb385afa0 100644 --- a/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb +++ b/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel') +Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel') diff --git a/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb b/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb index f6ea61f4502..4be61bfb689 100644 --- a/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb +++ b/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel') +Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel') diff --git a/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb b/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb index 95540cd5f49..1ea483f929f 100644 --- a/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb +++ b/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel') +Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel') diff --git a/lib/gitlab/background_migration/sync_blocking_issues_count.rb b/lib/gitlab/background_migration/sync_blocking_issues_count.rb index 6262320128c..49a632952fb 100644 --- a/lib/gitlab/background_migration/sync_blocking_issues_count.rb +++ b/lib/gitlab/background_migration/sync_blocking_issues_count.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::SyncBlockingIssuesCount.prepend_if_ee('EE::Gitlab::BackgroundMigration::SyncBlockingIssuesCount') +Gitlab::BackgroundMigration::SyncBlockingIssuesCount.prepend_mod_with('Gitlab::BackgroundMigration::SyncBlockingIssuesCount') diff --git a/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb b/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb index 651df36fcfd..054b918dade 100644 --- a/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb +++ b/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings') +Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings.prepend_mod_with('Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings') diff --git a/lib/gitlab/background_migration/update_timelogs_project_id.rb b/lib/gitlab/background_migration/update_timelogs_project_id.rb new file mode 100644 index 00000000000..24c9967b88e --- /dev/null +++ b/lib/gitlab/background_migration/update_timelogs_project_id.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class to populate project_id for timelogs + class UpdateTimelogsProjectId + BATCH_SIZE = 1000 + + def perform(start_id, stop_id) + (start_id..stop_id).step(BATCH_SIZE).each do |offset| + update_issue_timelogs(offset, offset + BATCH_SIZE) + update_merge_request_timelogs(offset, offset + BATCH_SIZE) + end + end + + def update_issue_timelogs(batch_start, batch_stop) + execute(<<~SQL) + UPDATE timelogs + SET project_id = issues.project_id + FROM issues + WHERE issues.id = timelogs.issue_id + AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop} + AND timelogs.project_id IS NULL; + SQL + end + + def update_merge_request_timelogs(batch_start, batch_stop) + execute(<<~SQL) + UPDATE timelogs + SET project_id = merge_requests.target_project_id + FROM merge_requests + WHERE merge_requests.id = timelogs.merge_request_id + AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop} + AND timelogs.project_id IS NULL; + SQL + end + + def execute(sql) + @connection ||= ::ActiveRecord::Base.connection + @connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb b/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb index bfe9f673b53..1cc03f061fb 100644 --- a/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb +++ b/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback') +Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback') diff --git a/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb b/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb index a2940cba6fa..60adb6b7e3e 100644 --- a/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb +++ b/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed') +Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed') diff --git a/lib/gitlab/background_migration/update_vulnerability_confidence.rb b/lib/gitlab/background_migration/update_vulnerability_confidence.rb index 6ffaa836f3c..40d29978dd4 100644 --- a/lib/gitlab/background_migration/update_vulnerability_confidence.rb +++ b/lib/gitlab/background_migration/update_vulnerability_confidence.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence') +Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence') diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb index a2b50c41f4a..d76d06606ee 100644 --- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb +++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb @@ -38,4 +38,4 @@ module Gitlab end end -Namespace.prepend_if_ee('::EE::Namespace') +Namespace.prepend_mod_with('Namespace') |