diff options
author | Robert Speicher <rspeicher@gmail.com> | 2021-01-20 13:34:23 -0600 |
---|---|---|
committer | Robert Speicher <rspeicher@gmail.com> | 2021-01-20 13:34:23 -0600 |
commit | 6438df3a1e0fb944485cebf07976160184697d72 (patch) | |
tree | 00b09bfd170e77ae9391b1a2f5a93ef6839f2597 /db/post_migrate | |
parent | 42bcd54d971da7ef2854b896a7b34f4ef8601067 (diff) | |
download | gitlab-ce-6438df3a1e0fb944485cebf07976160184697d72.tar.gz |
Add latest changes from gitlab-org/gitlab@13-8-stable-eev13.8.0-rc42
Diffstat (limited to 'db/post_migrate')
17 files changed, 352 insertions, 10 deletions
diff --git a/db/post_migrate/20190402224749_schedule_merge_request_assignees_migration_progress_check.rb b/db/post_migrate/20190402224749_schedule_merge_request_assignees_migration_progress_check.rb index 6fb67deb834..8ec6a4a24ec 100644 --- a/db/post_migrate/20190402224749_schedule_merge_request_assignees_migration_progress_check.rb +++ b/db/post_migrate/20190402224749_schedule_merge_request_assignees_migration_progress_check.rb @@ -3,7 +3,7 @@ class ScheduleMergeRequestAssigneesMigrationProgressCheck < ActiveRecord::Migration[5.0] include Gitlab::Database::MigrationHelpers - MIGRATION = 'MergeRequestAssigneesMigrationProgressCheck'.freeze + MIGRATION = 'MergeRequestAssigneesMigrationProgressCheck' DOWNTIME = false diff --git a/db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb b/db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb index cee99f7cf2e..845c855358b 100644 --- a/db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb +++ b/db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb @@ -20,8 +20,8 @@ class ScheduleSyncIssuablesStateIdWhereNil < ActiveRecord::Migration[5.1] # BATCH_SIZE = 5000 DELAY_INTERVAL = 120.seconds.to_i - ISSUES_MIGRATION = 'SyncIssuesStateId'.freeze - MERGE_REQUESTS_MIGRATION = 'SyncMergeRequestsStateId'.freeze + ISSUES_MIGRATION = 'SyncIssuesStateId' + MERGE_REQUESTS_MIGRATION = 'SyncMergeRequestsStateId' disable_ddl_transaction! diff --git a/db/post_migrate/20190611161642_add_index_to_events_and_audit_events_created_at_author_id.rb b/db/post_migrate/20190611161642_add_index_to_events_and_audit_events_created_at_author_id.rb index 342e83d6322..5b9afbe128c 100644 --- a/db/post_migrate/20190611161642_add_index_to_events_and_audit_events_created_at_author_id.rb +++ b/db/post_migrate/20190611161642_add_index_to_events_and_audit_events_created_at_author_id.rb @@ -7,9 +7,9 @@ class AddIndexToEventsAndAuditEventsCreatedAtAuthorId < ActiveRecord::Migration[ include Gitlab::Database::MigrationHelpers DOWNTIME = false - INDEX_NAME = 'analytics_index_%s_on_created_at_and_author_id'.freeze - EVENTS_INDEX_NAME = (INDEX_NAME % 'events').freeze - AUDIT_EVENTS_INDEX_NAME = (INDEX_NAME % 'audit_events').freeze + INDEX_NAME = 'analytics_index_%s_on_created_at_and_author_id' + EVENTS_INDEX_NAME = (INDEX_NAME % 'events') + AUDIT_EVENTS_INDEX_NAME = (INDEX_NAME % 'audit_events') disable_ddl_transaction! diff --git a/db/post_migrate/20191030223057_backfill_version_author_and_created_at.rb b/db/post_migrate/20191030223057_backfill_version_author_and_created_at.rb index 7a6d0b089a0..5fcec83bfc3 100644 --- a/db/post_migrate/20191030223057_backfill_version_author_and_created_at.rb +++ b/db/post_migrate/20191030223057_backfill_version_author_and_created_at.rb @@ -2,7 +2,7 @@ class BackfillVersionAuthorAndCreatedAt < ActiveRecord::Migration[5.2] DOWNTIME = false - MIGRATION = 'BackfillVersionDataFromGitaly'.freeze + MIGRATION = 'BackfillVersionDataFromGitaly' BATCH_SIZE = 500 disable_ddl_transaction! diff --git a/db/post_migrate/20191104142124_nullify_users_role.rb b/db/post_migrate/20191104142124_nullify_users_role.rb index 540d58e95ba..ab8eae46745 100644 --- a/db/post_migrate/20191104142124_nullify_users_role.rb +++ b/db/post_migrate/20191104142124_nullify_users_role.rb @@ -4,7 +4,7 @@ class NullifyUsersRole < ActiveRecord::Migration[5.2] include Gitlab::Database::MigrationHelpers disable_ddl_transaction! - INDEX_NAME = 'partial_index_users_updated_at_for_cleaning_mistaken_values'.freeze + INDEX_NAME = 'partial_index_users_updated_at_for_cleaning_mistaken_values' DOWNTIME = false diff --git a/db/post_migrate/20200207185149_schedule_fix_orphan_promoted_issues.rb b/db/post_migrate/20200207185149_schedule_fix_orphan_promoted_issues.rb index 83ba56501dd..d25d6ed2dc7 100644 --- a/db/post_migrate/20200207185149_schedule_fix_orphan_promoted_issues.rb +++ b/db/post_migrate/20200207185149_schedule_fix_orphan_promoted_issues.rb @@ -5,7 +5,7 @@ class ScheduleFixOrphanPromotedIssues < ActiveRecord::Migration[5.2] DOWNTIME = false BATCH_SIZE = 100 - BACKGROUND_MIGRATION = 'FixOrphanPromotedIssues'.freeze + BACKGROUND_MIGRATION = 'FixOrphanPromotedIssues' disable_ddl_transaction! diff --git a/db/post_migrate/20200217225719_schedule_migrate_security_scans.rb b/db/post_migrate/20200217225719_schedule_migrate_security_scans.rb index 7ef204ed9de..087c189f9fd 100644 --- a/db/post_migrate/20200217225719_schedule_migrate_security_scans.rb +++ b/db/post_migrate/20200217225719_schedule_migrate_security_scans.rb @@ -6,7 +6,7 @@ class ScheduleMigrateSecurityScans < ActiveRecord::Migration[5.2] DOWNTIME = false INTERVAL = 2.minutes.to_i BATCH_SIZE = 10_000 - MIGRATION = 'MigrateSecurityScans'.freeze + MIGRATION = 'MigrateSecurityScans' disable_ddl_transaction! diff --git a/db/post_migrate/20200716234259_remove_duplicate_labels_from_group.rb b/db/post_migrate/20200716234259_remove_duplicate_labels_from_group.rb new file mode 100644 index 00000000000..f19a209092b --- /dev/null +++ b/db/post_migrate/20200716234259_remove_duplicate_labels_from_group.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +class RemoveDuplicateLabelsFromGroup < ActiveRecord::Migration[6.0] + DOWNTIME = false + + CREATE = 1 + RENAME = 2 + + disable_ddl_transaction! + + class BackupLabel < ApplicationRecord + include EachBatch + + self.table_name = 'backup_labels' + end + + class Label < ApplicationRecord + self.table_name = 'labels' + end + + class Group < ApplicationRecord + include EachBatch + + self.table_name = 'namespaces' + end + + BATCH_SIZE = 10_000 + + def up + # Split to smaller chunks + # Loop rather than background job, every 10,000 + # there are ~1,800,000 groups in total (excluding personal namespaces, which can't have labels) + Group.where(type: 'Group').each_batch(of: BATCH_SIZE) do |batch| + range = batch.pluck('MIN(id)', 'MAX(id)').first + + transaction do + remove_full_duplicates(*range) + end + + transaction do + rename_partial_duplicates(*range) + end + end + end + + DOWN_BATCH_SIZE = 1000 + + def down + BackupLabel.where('project_id IS NULL AND group_id IS NOT NULL').each_batch(of: DOWN_BATCH_SIZE) do |batch| + range = batch.pluck('MIN(id)', 'MAX(id)').first + + restore_renamed_labels(*range) + restore_deleted_labels(*range) + end + end + + def remove_full_duplicates(start_id, stop_id) + # Fields that are considered duplicate: + # group_id title template description type color + + duplicate_labels = ApplicationRecord.connection.execute(<<-SQL.squish) +WITH data AS ( + SELECT labels.*, + row_number() OVER (PARTITION BY labels.group_id, labels.title, labels.template, labels.description, labels.type, labels.color ORDER BY labels.id) AS row_number, + #{CREATE} AS restore_action + FROM labels + WHERE labels.group_id BETWEEN #{start_id} AND #{stop_id} + AND NOT EXISTS (SELECT * FROM board_labels WHERE board_labels.label_id = labels.id) + AND NOT EXISTS (SELECT * FROM label_links WHERE label_links.label_id = labels.id) + AND NOT EXISTS (SELECT * FROM label_priorities WHERE label_priorities.label_id = labels.id) + AND NOT EXISTS (SELECT * FROM lists WHERE lists.label_id = labels.id) + AND NOT EXISTS (SELECT * FROM resource_label_events WHERE resource_label_events.label_id = labels.id) +) SELECT * FROM data WHERE row_number > 1; + SQL + + if duplicate_labels.any? + # create backup records + BackupLabel.insert_all!(duplicate_labels.map { |label| label.except("row_number") }) + + Label.unscoped.where(id: duplicate_labels.pluck("id")).delete_all + end + end + + def rename_partial_duplicates(start_id, stop_id) + # We need to ensure that the new title (with `_duplicate#{ID}`) doesn't exceed the limit. + # Truncate the original title (if needed) to 245 characters minus the length of the ID + # then add `_duplicate#{ID}` + + soft_duplicates = ApplicationRecord.connection.execute(<<-SQL.squish) +WITH data AS ( + SELECT + *, + substring(title from 1 for 245 - length(id::text)) || '_duplicate' || id::text as new_title, + #{RENAME} AS restore_action, + row_number() OVER (PARTITION BY group_id, title ORDER BY id) AS row_number + FROM labels + WHERE group_id BETWEEN #{start_id} AND #{stop_id} +) SELECT * FROM data WHERE row_number > 1; + SQL + + if soft_duplicates.any? + # create backup records + BackupLabel.insert_all!(soft_duplicates.map { |label| label.except("row_number") }) + + ApplicationRecord.connection.execute(<<-SQL.squish) +UPDATE labels SET title = substring(title from 1 for 245 - length(id::text)) || '_duplicate' || id::text +WHERE labels.id IN (#{soft_duplicates.map { |dup| dup["id"] }.join(", ")}); + SQL + end + end + + def restore_renamed_labels(start_id, stop_id) + # the backup label IDs are not incremental, they are copied directly from the Labels table + ApplicationRecord.connection.execute(<<-SQL.squish) +WITH backups AS ( + SELECT id, title + FROM backup_labels + WHERE id BETWEEN #{start_id} AND #{stop_id} + AND restore_action = #{RENAME} +) UPDATE labels SET title = backups.title +FROM backups +WHERE labels.id = backups.id; + SQL + end + + def restore_deleted_labels(start_id, stop_id) + ActiveRecord::Base.connection.execute(<<-SQL.squish) +INSERT INTO labels +SELECT id, title, color, group_id, created_at, updated_at, template, description, description_html, type, cached_markdown_version FROM backup_labels + WHERE backup_labels.id BETWEEN #{start_id} AND #{stop_id} + AND backup_labels.project_id IS NULL AND backup_labels.group_id IS NOT NULL + AND backup_labels.restore_action = #{CREATE} + SQL + end +end diff --git a/db/post_migrate/20200716234518_add_uniqueness_index_to_label_title_and_group.rb b/db/post_migrate/20200716234518_add_uniqueness_index_to_label_title_and_group.rb new file mode 100644 index 00000000000..08bab811c00 --- /dev/null +++ b/db/post_migrate/20200716234518_add_uniqueness_index_to_label_title_and_group.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class AddUniquenessIndexToLabelTitleAndGroup < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + GROUP_AND_TITLE = [:group_id, :title] + + def up + add_concurrent_index :labels, GROUP_AND_TITLE, where: "labels.project_id IS NULL", unique: true, name: "index_labels_on_group_id_and_title_unique" + remove_concurrent_index :labels, GROUP_AND_TITLE, name: "index_labels_on_group_id_and_title" + end + + def down + add_concurrent_index :labels, GROUP_AND_TITLE, where: "labels.project_id IS NULL", unique: false, name: "index_labels_on_group_id_and_title" + remove_concurrent_index :labels, GROUP_AND_TITLE, name: "index_labels_on_group_id_and_title_unique" + end +end diff --git a/db/post_migrate/20201207165956_remove_duplicate_services.rb b/db/post_migrate/20201207165956_remove_duplicate_services.rb new file mode 100644 index 00000000000..1659b9a2095 --- /dev/null +++ b/db/post_migrate/20201207165956_remove_duplicate_services.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class RemoveDuplicateServices < ActiveRecord::Migration[6.0] + DOWNTIME = false + + disable_ddl_transaction! + + def up + # noop, replaced by 20210112143418_remove_duplicate_services.rb + end + + def down + end +end diff --git a/db/post_migrate/20201208175117_schedule_backfilling_artifact_expiry_migration.rb b/db/post_migrate/20201208175117_schedule_backfilling_artifact_expiry_migration.rb new file mode 100644 index 00000000000..f11c0bbe33a --- /dev/null +++ b/db/post_migrate/20201208175117_schedule_backfilling_artifact_expiry_migration.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +class ScheduleBackfillingArtifactExpiryMigration < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + SWITCH_DATE = Time.utc(2020, 6, 22).freeze + INDEX_NAME = 'expired_artifacts_temp_index'.freeze + INDEX_CONDITION = "expire_at IS NULL AND created_at < '#{SWITCH_DATE}'" + + disable_ddl_transaction! + + class JobArtifact < ActiveRecord::Base + include EachBatch + + self.table_name = 'ci_job_artifacts' + + scope :without_expiry_date, -> { where(expire_at: nil) } + scope :before_switch, -> { where('created_at < ?', SWITCH_DATE) } + end + + def up + # Create temporary index for expired artifacts + # Needs to be removed in a later migration + add_concurrent_index(:ci_job_artifacts, %i(id created_at), where: INDEX_CONDITION, name: INDEX_NAME) + + queue_background_migration_jobs_by_range_at_intervals( + JobArtifact.without_expiry_date.before_switch, + ::Gitlab::BackgroundMigration::BackfillArtifactExpiryDate, + 2.minutes, + batch_size: 200_000 + ) + end + + def down + remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME + end +end diff --git a/db/post_migrate/20201211090634_schedule_populate_finding_uuid_for_vulnerability_feedback.rb b/db/post_migrate/20201211090634_schedule_populate_finding_uuid_for_vulnerability_feedback.rb new file mode 100644 index 00000000000..3ecb48dab0f --- /dev/null +++ b/db/post_migrate/20201211090634_schedule_populate_finding_uuid_for_vulnerability_feedback.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class SchedulePopulateFindingUuidForVulnerabilityFeedback < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + MIGRATION_CLASS = 'PopulateFindingUuidForVulnerabilityFeedback' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + + disable_ddl_transaction! + + def up + queue_background_migration_jobs_by_range_at_intervals( + Gitlab::BackgroundMigration::PopulateFindingUuidForVulnerabilityFeedback::VulnerabilityFeedback, + MIGRATION_CLASS, + DELAY_INTERVAL, + batch_size: BATCH_SIZE + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20201216185336_add_devops_adoption_snapshot_not_null.rb b/db/post_migrate/20201216185336_add_devops_adoption_snapshot_not_null.rb new file mode 100644 index 00000000000..1bb3c57f3cd --- /dev/null +++ b/db/post_migrate/20201216185336_add_devops_adoption_snapshot_not_null.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class AddDevopsAdoptionSnapshotNotNull < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + with_lock_retries do + execute( + <<~SQL + LOCK TABLE analytics_devops_adoption_snapshots IN ACCESS EXCLUSIVE MODE; + + UPDATE analytics_devops_adoption_snapshots SET end_time = date_trunc('month', recorded_at) - interval '1 millisecond'; + + ALTER TABLE analytics_devops_adoption_snapshots ALTER COLUMN end_time SET NOT NULL; + SQL + ) + end + end + + def down + with_lock_retries do + execute(<<~SQL) + ALTER TABLE analytics_devops_adoption_snapshots ALTER COLUMN end_time DROP NOT NULL; + SQL + end + end +end diff --git a/db/post_migrate/20201223012231_reindex_ci_pipelines_on_schedule_id_and_id.rb b/db/post_migrate/20201223012231_reindex_ci_pipelines_on_schedule_id_and_id.rb new file mode 100644 index 00000000000..9ed1aea911a --- /dev/null +++ b/db/post_migrate/20201223012231_reindex_ci_pipelines_on_schedule_id_and_id.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class ReindexCiPipelinesOnScheduleIdAndId < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + OLD_INDEX_NAME = 'index_ci_pipelines_on_pipeline_schedule_id' + NEW_INDEX_NAME = 'index_ci_pipelines_on_pipeline_schedule_id_and_id' + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_pipelines, [:pipeline_schedule_id, :id], name: NEW_INDEX_NAME + remove_concurrent_index_by_name :ci_pipelines, OLD_INDEX_NAME + end + + def down + add_concurrent_index :ci_pipelines, :pipeline_schedule_id, name: OLD_INDEX_NAME + remove_concurrent_index_by_name :ci_pipelines, NEW_INDEX_NAME + end +end diff --git a/db/post_migrate/20210105103649_delete_column_group_id_on_compliance_framework.rb b/db/post_migrate/20210105103649_delete_column_group_id_on_compliance_framework.rb new file mode 100644 index 00000000000..b13d2fe2d0a --- /dev/null +++ b/db/post_migrate/20210105103649_delete_column_group_id_on_compliance_framework.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class DeleteColumnGroupIdOnComplianceFramework < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def change + remove_column :compliance_management_frameworks, :group_id, :bigint + end +end diff --git a/db/post_migrate/20210107194543_remove_alerts_service_records.rb b/db/post_migrate/20210107194543_remove_alerts_service_records.rb new file mode 100644 index 00000000000..51a2f96ac7f --- /dev/null +++ b/db/post_migrate/20210107194543_remove_alerts_service_records.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class RemoveAlertsServiceRecords < ActiveRecord::Migration[6.0] + DOWNTIME = false + + disable_ddl_transaction! + + class Service < ActiveRecord::Base + self.table_name = 'services' + end + + def up + Service.delete_by(type: 'AlertsService') + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210112143418_remove_duplicate_services2.rb b/db/post_migrate/20210112143418_remove_duplicate_services2.rb new file mode 100644 index 00000000000..83d92a78473 --- /dev/null +++ b/db/post_migrate/20210112143418_remove_duplicate_services2.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# This replaces the previous post-deployment migration 20201207165956_remove_duplicate_services_spec.rb, +# we have to run this again due to a bug in how we were receiving the arguments in the background migration. +class RemoveDuplicateServices2 < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INTERVAL = 2.minutes + BATCH_SIZE = 5_000 + MIGRATION = 'RemoveDuplicateServices' + + disable_ddl_transaction! + + def up + project_ids_with_duplicates = Gitlab::BackgroundMigration::RemoveDuplicateServices::Service.project_ids_with_duplicates + + project_ids_with_duplicates.each_batch(of: BATCH_SIZE, column: :project_id) do |batch, index| + migrate_in( + INTERVAL * index, + MIGRATION, + batch.pluck(:project_id) + ) + end + end + + def down + end +end |