summaryrefslogtreecommitdiff
path: root/lib/gitlab/background_migration
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2022-09-19 23:18:09 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2022-09-19 23:18:09 +0000
commit6ed4ec3e0b1340f96b7c043ef51d1b33bbe85fde (patch)
treedc4d20fe6064752c0bd323187252c77e0a89144b /lib/gitlab/background_migration
parent9868dae7fc0655bd7ce4a6887d4e6d487690eeed (diff)
downloadgitlab-ce-6ed4ec3e0b1340f96b7c043ef51d1b33bbe85fde.tar.gz
Add latest changes from gitlab-org/gitlab@15-4-stable-eev15.4.0-rc42
Diffstat (limited to 'lib/gitlab/background_migration')
-rw-r--r--lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb31
-rw-r--r--lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb1
-rw-r--r--lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb2
-rw-r--r--lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb25
-rw-r--r--lib/gitlab/background_migration/backfill_project_repositories.rb4
-rw-r--r--lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb12
-rw-r--r--lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb32
-rw-r--r--lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb19
-rw-r--r--lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb13
-rw-r--r--lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb11
-rw-r--r--lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb9
-rw-r--r--lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb16
-rw-r--r--lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb12
-rw-r--r--lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb16
-rw-r--r--lib/gitlab/background_migration/destroy_invalid_group_members.rb23
-rw-r--r--lib/gitlab/background_migration/destroy_invalid_project_members.rb25
-rw-r--r--lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb29
-rw-r--r--lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb21
-rw-r--r--lib/gitlab/background_migration/mailers/unconfirm_mailer.rb2
-rw-r--r--lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb2
-rw-r--r--lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb43
-rw-r--r--lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb16
-rw-r--r--lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb28
-rw-r--r--lib/gitlab/background_migration/set_correct_vulnerability_state.rb7
24 files changed, 303 insertions, 96 deletions
diff --git a/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb
new file mode 100644
index 00000000000..2ee0594d0a6
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Backfills the `vulnerability_reads.casted_cluster_agent_id` column
+ class BackfillClusterAgentsHasVulnerabilities < Gitlab::BackgroundMigration::BatchedMigrationJob
+ VULNERABILITY_READS_JOIN = <<~SQL
+ INNER JOIN vulnerability_reads
+ ON vulnerability_reads.casted_cluster_agent_id = cluster_agents.id AND
+ vulnerability_reads.project_id = cluster_agents.project_id AND
+ vulnerability_reads.report_type = 7
+ SQL
+
+ RELATION = ->(relation) do
+ relation
+ .where(has_vulnerabilities: false)
+ end
+
+ def perform
+ each_sub_batch(
+ operation_name: :update_all,
+ batching_scope: RELATION
+ ) do |sub_batch|
+ sub_batch
+ .joins(VULNERABILITY_READS_JOIN)
+ .update_all(has_vulnerabilities: true)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb
index b9151343d6a..2d64b7378be 100644
--- a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb
+++ b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb
@@ -9,6 +9,7 @@ module Gitlab
# Migration only version of MergeRequest table
class MergeRequest < ::ApplicationRecord
include EachBatch
+ validates :suggested_reviewers, json_schema: { filename: 'merge_request_suggested_reviewers' }
CORRECTED_REGEXP_STR = "^(\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP)"
diff --git a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb
index 814f5a897a9..ce4c4a28b37 100644
--- a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb
+++ b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb
@@ -22,7 +22,7 @@ module Gitlab
ProjectFeature.connection.execute(
<<~SQL
UPDATE project_features pf
- SET package_registry_access_level = (CASE p.packages_enabled
+ SET package_registry_access_level = (CASE p.packages_enabled
WHEN true THEN (CASE p.visibility_level
WHEN #{PROJECT_PUBLIC} THEN #{FEATURE_PUBLIC}
WHEN #{PROJECT_INTERNAL} THEN #{FEATURE_ENABLED}
diff --git a/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb
new file mode 100644
index 00000000000..815c346bb39
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Back-fills the `issues.namespace_id` by setting it to corresponding project.project_namespace_id
+ class BackfillProjectNamespaceOnIssues < BatchedMigrationJob
+ def perform
+ each_sub_batch(
+ operation_name: :update_all,
+ batching_scope: -> (relation) {
+ relation.joins("INNER JOIN projects ON projects.id = issues.project_id")
+ .select("issues.id AS issue_id, projects.project_namespace_id").where(issues: { namespace_id: nil })
+ }
+ ) do |sub_batch|
+ connection.execute <<~SQL
+ UPDATE issues
+ SET namespace_id = projects.project_namespace_id
+ FROM (#{sub_batch.to_sql}) AS projects(issue_id, project_namespace_id)
+ WHERE issues.id = issue_id
+ SQL
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb
index 05e2ed72fb3..c49ef9d10f5 100644
--- a/lib/gitlab/background_migration/backfill_project_repositories.rb
+++ b/lib/gitlab/background_migration/backfill_project_repositories.rb
@@ -212,8 +212,8 @@ module Gitlab
def build_attributes_for_project(project)
{
project_id: project.id,
- shard_id: find_shard_id(project.repository_storage),
- disk_path: project.disk_path
+ shard_id: find_shard_id(project.repository_storage),
+ disk_path: project.disk_path
}
end
diff --git a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb
index 728b60f7a0e..0c41d6af209 100644
--- a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb
+++ b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb
@@ -10,16 +10,12 @@ module Gitlab
vulnerability_reads.project_id = cluster_agents.project_id
SQL
- RELATION = ->(relation) do
- relation
- .where(report_type: 7)
- end
+ CLUSTER_IMAGE_SCANNING_REPORT_TYPE = 7
+
+ scope_to ->(relation) { relation.where(report_type: CLUSTER_IMAGE_SCANNING_REPORT_TYPE) }
def perform
- each_sub_batch(
- operation_name: :update_all,
- batching_scope: RELATION
- ) do |sub_batch|
+ each_sub_batch(operation_name: :update_all) do |sub_batch|
sub_batch
.joins(CLUSTER_AGENTS_JOIN)
.update_all('casted_cluster_agent_id = CAST(vulnerability_reads.cluster_agent_id AS bigint)')
diff --git a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb
index 32962f2bb89..86d53ad798d 100644
--- a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb
+++ b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb
@@ -4,11 +4,9 @@ module Gitlab
module BackgroundMigration
# Backfills the `issues.work_item_type_id` column, replacing any
# instances of `NULL` with the appropriate `work_item_types.id` based on `issues.issue_type`
- class BackfillWorkItemTypeIdForIssues
+ class BackfillWorkItemTypeIdForIssues < BatchedMigrationJob
# Basic AR model for issues table
class MigrationIssue < ApplicationRecord
- include ::EachBatch
-
self.table_name = 'issues'
scope :base_query, ->(base_type) { where(work_item_type_id: nil, issue_type: base_type) }
@@ -16,29 +14,27 @@ module Gitlab
MAX_UPDATE_RETRIES = 3
- def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, base_type, base_type_id)
- parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id, base_type)
+ scope_to ->(relation) {
+ relation.where(issue_type: base_type)
+ }
+
+ job_arguments :base_type, :base_type_id
- parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
+ def perform
+ each_sub_batch(
+ operation_name: :update_all,
+ batching_scope: -> (relation) { relation.where(work_item_type_id: nil) }
+ ) do |sub_batch|
first, last = sub_batch.pick(Arel.sql('min(id), max(id)'))
# The query need to be reconstructed because .each_batch modifies the default scope
# See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510
reconstructed_sub_batch = MigrationIssue.unscoped.base_query(base_type).where(id: first..last)
- batch_metrics.time_operation(:update_all) do
- update_with_retry(reconstructed_sub_batch, base_type_id)
- end
-
- pause_ms = 0 if pause_ms < 0
- sleep(pause_ms * 0.001)
+ update_with_retry(reconstructed_sub_batch, base_type_id)
end
end
- def batch_metrics
- @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new
- end
-
private
# Retry mechanism required as update statements on the issues table will randomly take longer than
@@ -64,10 +60,6 @@ module Gitlab
def update_batch(sub_batch, base_type_id)
sub_batch.update_all(work_item_type_id: base_type_id)
end
-
- def relation_scoped_to_range(source_table, source_key_column, start_id, end_id, base_type)
- MigrationIssue.where(source_key_column => start_id..end_id).base_query(base_type)
- end
end
end
end
diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb
deleted file mode 100644
index 7d5fef67c25..00000000000
--- a/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- module BatchingStrategies
- # Batching class to use for back-filling issue's work_item_type_id for a single issue type.
- # Batches will be scoped to records where the foreign key is NULL and only of a given issue type
- #
- # If no more batches exist in the table, returns nil.
- class BackfillIssueWorkItemTypeBatchingStrategy < PrimaryKeyBatchingStrategy
- def apply_additional_filters(relation, job_arguments:, job_class: nil)
- issue_type = job_arguments.first
-
- relation.where(issue_type: issue_type)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb
index 9ad119310f7..72da2b5a2b7 100644
--- a/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb
+++ b/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb
@@ -3,18 +3,9 @@
module Gitlab
module BackgroundMigration
module BatchingStrategies
- # Batching class to use for back-filling project_statistic's container_registry_size.
- # Batches will be scoped to records where the project_ids are migrated
- #
- # If no more batches exist in the table, returns nil.
+ # Used to apply additional filters to the batching table, migrated to
+ # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93771
class BackfillProjectStatisticsWithContainerRegistrySizeBatchingStrategy < PrimaryKeyBatchingStrategy
- MIGRATION_PHASE_1_ENDED_AT = Date.new(2022, 01, 23).freeze
-
- def apply_additional_filters(relation, job_arguments: [], job_class: nil)
- relation.where(created_at: MIGRATION_PHASE_1_ENDED_AT..).or(
- relation.where(migration_state: 'import_done')
- ).select(:project_id).distinct
- end
end
end
end
diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb
index f0d015198dc..c2fa00f66de 100644
--- a/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb
+++ b/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb
@@ -3,16 +3,9 @@
module Gitlab
module BackgroundMigration
module BatchingStrategies
- # Batching class to use for back-filling vulnerability_read's casted_cluster_agent_id from cluster_agent_id.
- # Batches will be scoped to records where the report_type belongs to cluster_image_scanning.
- #
- # If no more batches exist in the table, returns nil.
+ # Used to apply additional filters to the batching table, migrated to
+ # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93771
class BackfillVulnerabilityReadsClusterAgentBatchingStrategy < PrimaryKeyBatchingStrategy
- CLUSTER_IMAGE_SCANNING_REPORT_TYPE = 7
-
- def apply_additional_filters(relation, job_arguments: [], job_class: nil)
- relation.where(report_type: CLUSTER_IMAGE_SCANNING_REPORT_TYPE)
- end
end
end
end
diff --git a/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb b/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb
index e1855b6cfee..9504d4eec11 100644
--- a/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb
+++ b/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb
@@ -3,14 +3,9 @@
module Gitlab
module BackgroundMigration
module BatchingStrategies
- # Batching class to use for setting state in vulnerabilitites table.
- # Batches will be scoped to records where the dismissed_at is set.
- #
- # If no more batches exist in the table, returns nil.
+ # Used to apply additional filters to the batching table, migrated to
+ # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93771
class DismissedVulnerabilitiesStrategy < PrimaryKeyBatchingStrategy
- def apply_additional_filters(relation, job_arguments: [], job_class: nil)
- relation.where.not(dismissed_at: nil)
- end
end
end
end
diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb
index 1ffa4a052e5..43352b1bf91 100644
--- a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb
+++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb
@@ -22,8 +22,8 @@ module Gitlab
def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:, job_class: nil)
model_class = define_batchable_model(table_name, connection: connection)
- quoted_column_name = model_class.connection.quote_column_name(column_name)
- relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value)
+ arel_column = model_class.arel_table[column_name]
+ relation = model_class.where(arel_column.gteq(batch_min_value))
if job_class
relation = filter_batch(relation,
@@ -32,11 +32,10 @@ module Gitlab
)
end
- relation = apply_additional_filters(relation, job_arguments: job_arguments, job_class: job_class)
next_batch_bounds = nil
relation.each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop
- next_batch_bounds = batch.pick(Arel.sql("MIN(#{quoted_column_name}), MAX(#{quoted_column_name})"))
+ next_batch_bounds = batch.pick(arel_column.minimum, arel_column.maximum)
break
end
@@ -44,15 +43,6 @@ module Gitlab
next_batch_bounds
end
- # Deprecated
- #
- # Use `scope_to` to define additional filters on the migration job class.
- #
- # see https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#adding-additional-filters.
- def apply_additional_filters(relation, job_arguments: [], job_class: nil)
- relation
- end
-
private
def filter_batch(relation, table_name:, column_name:, job_class:, job_arguments: [])
diff --git a/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb
new file mode 100644
index 00000000000..49525479637
--- /dev/null
+++ b/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module BatchingStrategies
+ # Used to apply additional filters to the batching table, migrated to
+ # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96478
+ class RemoveBackfilledJobArtifactsExpireAtBatchingStrategy < PrimaryKeyBatchingStrategy
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb b/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb
new file mode 100644
index 00000000000..739197898d9
--- /dev/null
+++ b/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # This class doesn't delete approval rules
+ # as this feature exists only in EE
+ class DeleteApprovalRulesWithVulnerability < BatchedMigrationJob
+ def perform
+ end
+ end
+ end
+end
+
+# rubocop:disable Layout/LineLength
+Gitlab::BackgroundMigration::DeleteApprovalRulesWithVulnerability.prepend_mod_with('Gitlab::BackgroundMigration::DeleteApprovalRulesWithVulnerability')
+# rubocop:enable Layout/LineLength
diff --git a/lib/gitlab/background_migration/destroy_invalid_group_members.rb b/lib/gitlab/background_migration/destroy_invalid_group_members.rb
new file mode 100644
index 00000000000..35ac42f76ab
--- /dev/null
+++ b/lib/gitlab/background_migration/destroy_invalid_group_members.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ class DestroyInvalidGroupMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation
+ scope_to ->(relation) do
+ relation.where(source_type: 'Namespace')
+ .joins('LEFT OUTER JOIN namespaces ON members.source_id = namespaces.id')
+ .where(namespaces: { id: nil })
+ end
+
+ def perform
+ each_sub_batch(operation_name: :delete_all) do |sub_batch|
+ invalid_ids = sub_batch.map(&:id)
+ Gitlab::AppLogger.info({ message: 'Removing invalid group member records',
+ deleted_count: invalid_ids.size, ids: invalid_ids })
+
+ sub_batch.delete_all
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/destroy_invalid_project_members.rb b/lib/gitlab/background_migration/destroy_invalid_project_members.rb
new file mode 100644
index 00000000000..3c60f765c29
--- /dev/null
+++ b/lib/gitlab/background_migration/destroy_invalid_project_members.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ class DestroyInvalidProjectMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation
+ scope_to ->(relation) { relation.where(source_type: 'Project') }
+
+ def perform
+ each_sub_batch(operation_name: :delete_all) do |sub_batch|
+ invalid_project_members = sub_batch
+ .joins('LEFT OUTER JOIN projects ON members.source_id = projects.id')
+ .where(projects: { id: nil })
+ invalid_ids = invalid_project_members.pluck(:id)
+
+ # the actual delete
+ deleted_count = invalid_project_members.delete_all
+
+ Gitlab::AppLogger.info({ message: 'Removing invalid project member records',
+ deleted_count: deleted_count,
+ ids: invalid_ids })
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb
new file mode 100644
index 00000000000..824054b31f2
--- /dev/null
+++ b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Set `project_settings.legacy_open_source_license_available` to false for public projects created after 17/02/2022
+ class DisableLegacyOpenSourceLicenceForRecentPublicProjects < ::Gitlab::BackgroundMigration::BatchedMigrationJob
+ PUBLIC = 20
+ THRESHOLD_DATE = '2022-02-17 09:00:00'
+
+ # Migration only version of `project_settings` table
+ class ProjectSetting < ApplicationRecord
+ self.table_name = 'project_settings'
+ end
+
+ def perform
+ each_sub_batch(
+ operation_name: :disable_legacy_open_source_licence_for_recent_public_projects,
+ batching_scope: ->(relation) {
+ relation.where(visibility_level: PUBLIC).where('created_at >= ?', THRESHOLD_DATE)
+ }
+ ) do |sub_batch|
+ ProjectSetting.where(project_id: sub_batch)
+ .where(legacy_open_source_license_available: true)
+ .update_all(legacy_open_source_license_available: false)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb
new file mode 100644
index 00000000000..6e4d5d8ddcb
--- /dev/null
+++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Set `project_settings.legacy_open_source_license_available` to false for projects less than 1 MB
+ class DisableLegacyOpenSourceLicenseForProjectsLessThanOneMb < ::Gitlab::BackgroundMigration::BatchedMigrationJob
+ scope_to ->(relation) { relation.where(legacy_open_source_license_available: true) }
+
+ def perform
+ each_sub_batch(operation_name: :disable_legacy_open_source_license_for_projects_less_than_one_mb) do |sub_batch|
+ updates = { legacy_open_source_license_available: false, updated_at: Time.current }
+
+ sub_batch
+ .joins('INNER JOIN project_statistics ON project_statistics.project_id = project_settings.project_id')
+ .where('project_statistics.repository_size < ?', 1.megabyte)
+ .update_all(updates)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb
index 3605b157f4f..2bf631c6c7d 100644
--- a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb
+++ b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb
@@ -11,7 +11,7 @@ module Gitlab
@user = user
@verification_from_mail = Gitlab.config.gitlab.email_from
- mail(
+ mail_with_locale(
template_path: 'unconfirm_mailer',
template_name: 'unconfirm_notification_email',
to: @user.notification_email_or_default,
diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
index 72380af2c53..9a42d035285 100644
--- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
+++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
@@ -58,7 +58,7 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # r
development: "a143e9e2-41b3-47bc-9a19-081d089229f4",
test: "a143e9e2-41b3-47bc-9a19-081d089229f4",
staging: "a6930898-a1b2-4365-ab18-12aa474d9b26",
- production: "58dc0f06-936c-43b3-93bb-71693f1b6570"
+ production: "58dc0f06-936c-43b3-93bb-71693f1b6570"
}.freeze
NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze
diff --git a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb
new file mode 100644
index 00000000000..d30263976e8
--- /dev/null
+++ b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # This detects and fixes job artifacts that have `expire_at` wrongly backfilled by the migration
+ # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723.
+ # These job artifacts will not be deleted and will have their `expire_at` removed.
+ class RemoveBackfilledJobArtifactsExpireAt < BatchedMigrationJob
+ # The migration would have backfilled `expire_at`
+ # to midnight on the 22nd of the month of the local timezone,
+ # storing it as UTC time in the database.
+ #
+ # If the timezone setting has changed since the migration,
+ # the `expire_at` stored in the database could have changed to a different local time other than midnight.
+ # For example:
+ # - changing timezone from UTC+02:00 to UTC+02:30 would change the `expire_at` in local time 00:00:00 to 00:30:00.
+ # - changing timezone from UTC+00:00 to UTC-01:00 would change the `expire_at` in local time 00:00:00 to 23:00:00
+ # on the previous day (21st).
+ #
+ # Therefore job artifacts that have `expire_at` exactly on the 00, 30 or 45 minute mark
+ # on the dates 21, 22, 23 of the month will not be deleted.
+ # https://en.wikipedia.org/wiki/List_of_UTC_time_offsets
+ EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE = <<~SQL
+ EXTRACT(day FROM timezone('UTC', expire_at)) IN (21, 22, 23)
+ AND EXTRACT(minute FROM timezone('UTC', expire_at)) IN (0, 30, 45)
+ AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0
+ SQL
+
+ scope_to ->(relation) {
+ relation.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE)
+ .or(relation.where(file_type: 3))
+ }
+
+ def perform
+ each_sub_batch(
+ operation_name: :update_all
+ ) do |sub_batch|
+ sub_batch.update_all(expire_at: nil)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb
new file mode 100644
index 00000000000..5b1d630bb03
--- /dev/null
+++ b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Removes obsolete wiki notes
+ class RemoveSelfManagedWikiNotes < BatchedMigrationJob
+ def perform
+ each_sub_batch(
+ operation_name: :delete_all
+ ) do |sub_batch|
+ sub_batch.where(noteable_type: 'Wiki').delete_all
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb
new file mode 100644
index 00000000000..718fb0aaa71
--- /dev/null
+++ b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Renames all system notes created when an issuable task is checked/unchecked
+ # from `task` into `checklist item`
+ # `marked the task **Task 1** as incomplete` => `marked the checklist item **Task 1** as incomplete`
+ class RenameTaskSystemNoteToChecklistItem < BatchedMigrationJob
+ REPLACE_REGEX = '\Amarked\sthe\stask'
+ TEXT_REPLACEMENT = 'marked the checklist item'
+
+ scope_to ->(relation) {
+ relation.where(system_note_metadata: { action: :task })
+ }
+
+ def perform
+ each_sub_batch(operation_name: :update_all) do |sub_batch|
+ ApplicationRecord.connection.execute <<~SQL
+ UPDATE notes
+ SET note = REGEXP_REPLACE(notes.note,'#{REPLACE_REGEX}', '#{TEXT_REPLACEMENT}')
+ FROM (#{sub_batch.select(:note_id).to_sql}) AS metadata_fields(note_id)
+ WHERE notes.id = note_id
+ SQL
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb
index fd6cbcb8d05..a0cfeed618a 100644
--- a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb
+++ b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb
@@ -6,11 +6,10 @@ module Gitlab
class SetCorrectVulnerabilityState < BatchedMigrationJob
DISMISSED_STATE = 2
+ scope_to ->(relation) { relation.where.not(dismissed_at: nil) }
+
def perform
- each_sub_batch(
- operation_name: :update_vulnerabilities_state,
- batching_scope: -> (relation) { relation.where.not(dismissed_at: nil) }
- ) do |sub_batch|
+ each_sub_batch(operation_name: :update_vulnerabilities_state) do |sub_batch|
sub_batch.update_all(state: DISMISSED_STATE)
end
end