diff options
Diffstat (limited to 'db/post_migrate')
20 files changed, 560 insertions, 4 deletions
diff --git a/db/post_migrate/20201015073808_schedule_blocked_by_links_replacement.rb b/db/post_migrate/20201015073808_schedule_blocked_by_links_replacement.rb index 7833d7c4c04..dd8cbb57136 100644 --- a/db/post_migrate/20201015073808_schedule_blocked_by_links_replacement.rb +++ b/db/post_migrate/20201015073808_schedule_blocked_by_links_replacement.rb @@ -19,10 +19,8 @@ class ScheduleBlockedByLinksReplacement < ActiveRecord::Migration[6.0] end def up - relation = IssueLink.where(link_type: 2) - - queue_background_migration_jobs_by_range_at_intervals( - relation, MIGRATION, INTERVAL, batch_size: BATCH_SIZE) + # no-op + # superseded by db/post_migrate/20201102073808_schedule_blocked_by_links_replacement_second_try.rb end def down diff --git a/db/post_migrate/20201019094741_rename_sitemap_root_namespaces.rb b/db/post_migrate/20201019094741_rename_sitemap_root_namespaces.rb new file mode 100644 index 00000000000..20812a53bfb --- /dev/null +++ b/db/post_migrate/20201019094741_rename_sitemap_root_namespaces.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class RenameSitemapRootNamespaces < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + include Gitlab::Database::RenameReservedPathsMigration::V1 + + DOWNTIME = false + + disable_ddl_transaction! + + # We're taking over the /sitemap.xml and /sitemap.xml.gz namespaces + # since they're necessary for the default behavior of Sitemaps + def up + disable_statement_timeout do + rename_root_paths(['sitemap.xml', 'sitemap.xml.gz']) + end + end + + def down + disable_statement_timeout do + revert_renames + end + end +end diff --git a/db/post_migrate/20201020102551_remove_index_service_for_usage_data.rb b/db/post_migrate/20201020102551_remove_index_service_for_usage_data.rb new file mode 100644 index 00000000000..a0d39ecd2c1 --- /dev/null +++ b/db/post_migrate/20201020102551_remove_index_service_for_usage_data.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class RemoveIndexServiceForUsageData < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_services_on_type_id_when_active_not_instance_not_template' + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name :services, INDEX_NAME + end + + def down + add_concurrent_index :services, [:type, :id], where: 'active = TRUE AND instance = FALSE AND template = FALSE', name: INDEX_NAME + end +end diff --git a/db/post_migrate/20201026051643_remove_scanned_resources_count_from_security_scans.rb b/db/post_migrate/20201026051643_remove_scanned_resources_count_from_security_scans.rb new file mode 100644 index 00000000000..208448e2278 --- /dev/null +++ b/db/post_migrate/20201026051643_remove_scanned_resources_count_from_security_scans.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class RemoveScannedResourcesCountFromSecurityScans < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + remove_column :security_scans, :scanned_resources_count + end + + def down + add_column :security_scans, :scanned_resources_count, :integer + end +end diff --git a/db/post_migrate/20201026182253_schedule_populate_vulnerability_feedback_pipeline_id.rb b/db/post_migrate/20201026182253_schedule_populate_vulnerability_feedback_pipeline_id.rb new file mode 100644 index 00000000000..2df475ab2a8 --- /dev/null +++ b/db/post_migrate/20201026182253_schedule_populate_vulnerability_feedback_pipeline_id.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class SchedulePopulateVulnerabilityFeedbackPipelineId < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INTERVAL = 2.minutes.to_i + BATCH_SIZE = 100 + MIGRATION = 'PopulateVulnerabilityFeedbackPipelineId' + + disable_ddl_transaction! + + def up + return unless Gitlab.ee? + + vulnerability_feedback = exec_query <<~SQL + SELECT DISTINCT "vulnerability_feedback"."project_id" + FROM "vulnerability_feedback" + WHERE "vulnerability_feedback"."pipeline_id" IS NULL + ORDER BY "vulnerability_feedback"."project_id" ASC + SQL + + return if vulnerability_feedback.rows.blank? + + vulnerability_feedback.rows.flatten.in_groups_of(BATCH_SIZE, false).each_with_index do |project_ids, index| + migrate_in(index * INTERVAL, MIGRATION, [project_ids]) + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20201028160832_schedule_populate_missing_dismissal_information_for_vulnerabilities.rb b/db/post_migrate/20201028160832_schedule_populate_missing_dismissal_information_for_vulnerabilities.rb new file mode 100644 index 00000000000..f358ea863db --- /dev/null +++ b/db/post_migrate/20201028160832_schedule_populate_missing_dismissal_information_for_vulnerabilities.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class SchedulePopulateMissingDismissalInformationForVulnerabilities < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 1_000 + DELAY_INTERVAL = 3.minutes.to_i + MIGRATION_CLASS = 'PopulateMissingVulnerabilityDismissalInformation' + + disable_ddl_transaction! + + def up + ::Gitlab::BackgroundMigration::PopulateMissingVulnerabilityDismissalInformation::Vulnerability.broken.each_batch(of: BATCH_SIZE) do |batch, index| + vulnerability_ids = batch.pluck(:id) + migrate_in(index * DELAY_INTERVAL, MIGRATION_CLASS, vulnerability_ids) + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20201028182809_backfill_jira_tracker_deployment_type2.rb b/db/post_migrate/20201028182809_backfill_jira_tracker_deployment_type2.rb new file mode 100644 index 00000000000..63a0554433c --- /dev/null +++ b/db/post_migrate/20201028182809_backfill_jira_tracker_deployment_type2.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class BackfillJiraTrackerDeploymentType2 < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + MIGRATION = 'BackfillJiraTrackerDeploymentType2' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + + disable_ddl_transaction! + + class JiraTrackerData < ActiveRecord::Base + include EachBatch + + self.table_name = 'jira_tracker_data' + end + + def up + queue_background_migration_jobs_by_range_at_intervals( + JiraTrackerData.where(deployment_type: 0), + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + track_jobs: true) + end + + def down + # NOOP + end +end diff --git a/db/post_migrate/20201029052241_migrate_geo_blob_verification_primary_worker_sidekiq_queue.rb b/db/post_migrate/20201029052241_migrate_geo_blob_verification_primary_worker_sidekiq_queue.rb new file mode 100644 index 00000000000..64d22863389 --- /dev/null +++ b/db/post_migrate/20201029052241_migrate_geo_blob_verification_primary_worker_sidekiq_queue.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class MigrateGeoBlobVerificationPrimaryWorkerSidekiqQueue < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + sidekiq_queue_migrate 'geo:geo_blob_verification_primary', to: 'geo:geo_verification' + end + + def down + sidekiq_queue_migrate 'geo:geo_verification', to: 'geo:geo_blob_verification_primary' + end +end diff --git a/db/post_migrate/20201029144157_cleanup_application_settings_to_allow_deny_rename.rb b/db/post_migrate/20201029144157_cleanup_application_settings_to_allow_deny_rename.rb new file mode 100644 index 00000000000..bb6fe4258c5 --- /dev/null +++ b/db/post_migrate/20201029144157_cleanup_application_settings_to_allow_deny_rename.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class CleanupApplicationSettingsToAllowDenyRename < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + cleanup_concurrent_column_rename :application_settings, :domain_blacklist_enabled, :domain_denylist_enabled + cleanup_concurrent_column_rename :application_settings, :domain_blacklist, :domain_denylist + cleanup_concurrent_column_rename :application_settings, :domain_whitelist, :domain_allowlist + end + + def down + undo_cleanup_concurrent_column_rename :application_settings, :domain_blacklist_enabled, :domain_denylist_enabled + undo_cleanup_concurrent_column_rename :application_settings, :domain_blacklist, :domain_denylist + undo_cleanup_concurrent_column_rename :application_settings, :domain_whitelist, :domain_allowlist + end +end diff --git a/db/post_migrate/20201030203854_backfill_design_iids.rb b/db/post_migrate/20201030203854_backfill_design_iids.rb new file mode 100644 index 00000000000..7acca6ad93d --- /dev/null +++ b/db/post_migrate/20201030203854_backfill_design_iids.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class BackfillDesignIids < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class Designs < ActiveRecord::Base + include EachBatch + + self.table_name = 'design_management_designs' + end + + def up + backfill = ::Gitlab::BackgroundMigration::BackfillDesignInternalIds.new(Designs) + + Designs.select(:project_id).distinct.each_batch(of: 100, column: :project_id) do |relation| + backfill.perform(relation) + end + end + + def down + # NOOP + end +end diff --git a/db/post_migrate/20201102073808_schedule_blocked_by_links_replacement_second_try.rb b/db/post_migrate/20201102073808_schedule_blocked_by_links_replacement_second_try.rb new file mode 100644 index 00000000000..217d4f81d26 --- /dev/null +++ b/db/post_migrate/20201102073808_schedule_blocked_by_links_replacement_second_try.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class ScheduleBlockedByLinksReplacementSecondTry < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INTERVAL = 2.minutes + # at the time of writing there were 12931 blocked_by issues: + # estimated time is 13 batches * 2 minutes -> 26 minutes + BATCH_SIZE = 1000 + MIGRATION = 'ReplaceBlockedByLinks' + + disable_ddl_transaction! + + class IssueLink < ActiveRecord::Base + include EachBatch + + self.table_name = 'issue_links' + end + + def up + relation = IssueLink.where(link_type: 2) + + queue_background_migration_jobs_by_range_at_intervals( + relation, MIGRATION, INTERVAL, batch_size: BATCH_SIZE) + end + + def down + end +end diff --git a/db/post_migrate/20201102112206_rename_sitemap_namespace.rb b/db/post_migrate/20201102112206_rename_sitemap_namespace.rb new file mode 100644 index 00000000000..b2e610d68db --- /dev/null +++ b/db/post_migrate/20201102112206_rename_sitemap_namespace.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class RenameSitemapNamespace < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + include Gitlab::Database::RenameReservedPathsMigration::V1 + + DOWNTIME = false + + disable_ddl_transaction! + + # We're taking over the /sitemap namespace + # since it's necessary for the default behavior of Sitemaps + def up + disable_statement_timeout do + rename_root_paths(['sitemap']) + end + end + + def down + disable_statement_timeout do + revert_renames + end + end +end diff --git a/db/post_migrate/20201102114018_remove_storage_size_limit_from_application_settings.rb b/db/post_migrate/20201102114018_remove_storage_size_limit_from_application_settings.rb new file mode 100644 index 00000000000..6646cf2ad0c --- /dev/null +++ b/db/post_migrate/20201102114018_remove_storage_size_limit_from_application_settings.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class RemoveStorageSizeLimitFromApplicationSettings < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + remove_column :application_settings, :namespace_storage_size_limit + end + + def down + add_column :application_settings, :namespace_storage_size_limit, :bigint, default: 0 + end +end diff --git a/db/post_migrate/20201102152554_add_not_null_check_on_iid_on_design_manangement_designs.rb b/db/post_migrate/20201102152554_add_not_null_check_on_iid_on_design_manangement_designs.rb new file mode 100644 index 00000000000..861a0c3c27a --- /dev/null +++ b/db/post_migrate/20201102152554_add_not_null_check_on_iid_on_design_manangement_designs.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddNotNullCheckOnIidOnDesignManangementDesigns < ActiveRecord::Migration[6.0] + include ::Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_not_null_constraint(:design_management_designs, :iid) + end + + def down + remove_not_null_constraint(:design_management_designs, :iid) + end +end diff --git a/db/post_migrate/20201102152945_truncate_security_findings_table.rb b/db/post_migrate/20201102152945_truncate_security_findings_table.rb new file mode 100644 index 00000000000..8bfaa31f0de --- /dev/null +++ b/db/post_migrate/20201102152945_truncate_security_findings_table.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class TruncateSecurityFindingsTable < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + return unless Gitlab.dev_env_or_com? + + with_lock_retries do + connection.execute('TRUNCATE security_findings RESTART IDENTITY') + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20201103013242_remove_terraform_state_verification_columns.rb b/db/post_migrate/20201103013242_remove_terraform_state_verification_columns.rb new file mode 100644 index 00000000000..b1cd9790ead --- /dev/null +++ b/db/post_migrate/20201103013242_remove_terraform_state_verification_columns.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class RemoveTerraformStateVerificationColumns < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + transaction do + remove_column :terraform_states, :verification_retry_at, :datetime_with_timezone + remove_column :terraform_states, :verified_at, :datetime_with_timezone + remove_column :terraform_states, :verification_retry_count, :integer, limit: 2 + remove_column :terraform_states, :verification_checksum, :binary, using: 'verification_checksum::bytea' + remove_column :terraform_states, :verification_failure, :text + end + end + + def down + add_column(:terraform_states, :verification_retry_at, :datetime_with_timezone) unless column_exists?(:terraform_states, :verification_retry_at) + add_column(:terraform_states, :verified_at, :datetime_with_timezone) unless column_exists?(:terraform_states, :verified_at) + add_column(:terraform_states, :verification_retry_count, :integer, limit: 2) unless column_exists?(:terraform_states, :verification_retry_count) + add_column(:terraform_states, :verification_checksum, :binary, using: 'verification_checksum::bytea') unless column_exists?(:terraform_states, :verification_checksum) + add_column(:terraform_states, :verification_failure, :text) unless column_exists?(:terraform_states, :verification_failure) + + add_text_limit :terraform_states, :verification_failure, 255 + end +end diff --git a/db/post_migrate/20201103110018_schedule_merge_request_cleanup_schedules_backfill.rb b/db/post_migrate/20201103110018_schedule_merge_request_cleanup_schedules_backfill.rb new file mode 100644 index 00000000000..77057205b09 --- /dev/null +++ b/db/post_migrate/20201103110018_schedule_merge_request_cleanup_schedules_backfill.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class ScheduleMergeRequestCleanupSchedulesBackfill < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + MIGRATION = 'BackfillMergeRequestCleanupSchedules' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 10_000 + TEMP_INDEX_NAME = 'merge_requests_state_id_temp_index' + + disable_ddl_transaction! + + def up + add_concurrent_index :merge_requests, :id, name: TEMP_INDEX_NAME, where: "state_id IN (2, 3)" + + eligible_mrs = Gitlab::BackgroundMigration::BackfillMergeRequestCleanupSchedules::MergeRequest.eligible + + queue_background_migration_jobs_by_range_at_intervals( + eligible_mrs, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE + ) + end + + def down + remove_concurrent_index_by_name :merge_requests, TEMP_INDEX_NAME + end +end diff --git a/db/post_migrate/20201103192526_schedule_populate_has_vulnerabilities.rb b/db/post_migrate/20201103192526_schedule_populate_has_vulnerabilities.rb new file mode 100644 index 00000000000..bed90af09dc --- /dev/null +++ b/db/post_migrate/20201103192526_schedule_populate_has_vulnerabilities.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class SchedulePopulateHasVulnerabilities < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 1_000 + DELAY_INTERVAL = 2.minutes + MIGRATION_CLASS = 'PopulateHasVulnerabilities' + + disable_ddl_transaction! + + def up + Gitlab::BackgroundMigration::PopulateHasVulnerabilities::Vulnerability.distinct.each_batch(of: BATCH_SIZE, column: :project_id) do |batch, index| + project_ids = batch.pluck(:project_id) + + migrate_in(index * DELAY_INTERVAL, MIGRATION_CLASS, project_ids) + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20201106082723_add_merge_request_jira_reference_indexes.rb b/db/post_migrate/20201106082723_add_merge_request_jira_reference_indexes.rb new file mode 100644 index 00000000000..d69f57c6088 --- /dev/null +++ b/db/post_migrate/20201106082723_add_merge_request_jira_reference_indexes.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +class AddMergeRequestJiraReferenceIndexes < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + DESCRIPTION_INDEX_NAME = 'index_merge_requests_on_target_project_id_iid_jira_description' + TITLE_INDEX_NAME = 'index_merge_requests_on_target_project_id_and_iid_jira_title' + + JIRA_KEY_REGEX = '[A-Z][A-Z_0-9]+-\d+' + + disable_ddl_transaction! + + def up + add_concurrent_index( + :merge_requests, + [:target_project_id, :iid], + name: TITLE_INDEX_NAME, + using: :btree, + where: "(merge_requests.title)::text ~ '#{JIRA_KEY_REGEX}'::text" + ) + + add_concurrent_index( + :merge_requests, + [:target_project_id, :iid], + name: DESCRIPTION_INDEX_NAME, + using: :btree, + where: "(merge_requests.description)::text ~ '#{JIRA_KEY_REGEX}'::text" + ) + end + + def down + remove_concurrent_index_by_name( + :merge_requests, + TITLE_INDEX_NAME + ) + + remove_concurrent_index_by_name( + :merge_requests, + DESCRIPTION_INDEX_NAME + ) + end +end diff --git a/db/post_migrate/20201106134950_deduplicate_epic_iids.rb b/db/post_migrate/20201106134950_deduplicate_epic_iids.rb new file mode 100644 index 00000000000..bc7daf9329d --- /dev/null +++ b/db/post_migrate/20201106134950_deduplicate_epic_iids.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +class DeduplicateEpicIids < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_epics_on_group_id_and_iid' + + disable_ddl_transaction! + + class Epic < ActiveRecord::Base + end + + class InternalId < ActiveRecord::Base + class << self + def generate_next(subject, scope, usage, init) + InternalIdGenerator.new(subject, scope, usage, init).generate + end + end + + # Increments #last_value and saves the record + # + # The operation locks the record and gathers a `ROW SHARE` lock (in PostgreSQL). + # As such, the increment is atomic and safe to be called concurrently. + def increment_and_save! + update_and_save { self.last_value = (last_value || 0) + 1 } + end + + private + + def update_and_save(&block) + lock! + yield + save! + last_value + end + end + + # See app/models/internal_id + class InternalIdGenerator + attr_reader :subject, :scope, :scope_attrs, :usage, :init + + def initialize(subject, scope, usage, init = nil) + @subject = subject + @scope = scope + @usage = usage + @init = init + + raise ArgumentError, 'Scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty? || usage.to_s != 'epics' + end + + # Generates next internal id and returns it + # init: Block that gets called to initialize InternalId record if not present + # Make sure to not throw exceptions in the absence of records (if this is expected). + def generate + subject.transaction do + # Create a record in internal_ids if one does not yet exist + # and increment its last value + # + # Note this will acquire a ROW SHARE lock on the InternalId record + record.increment_and_save! + end + end + + def record + @record ||= (lookup || create_record) + end + + def lookup + InternalId.find_by(**scope, usage: usage_value) + end + + def usage_value + 4 # see Enums::InternalId - this is the value for epics + end + + # Create InternalId record for (scope, usage) combination, if it doesn't exist + # + # We blindly insert without synchronization. If another process + # was faster in doing this, we'll realize once we hit the unique key constraint + # violation. We can safely roll-back the nested transaction and perform + # a lookup instead to retrieve the record. + def create_record + raise ArgumentError, 'Cannot initialize without init!' unless init + + instance = subject.is_a?(::Class) ? nil : subject + + subject.transaction(requires_new: true) do + InternalId.create!( + **scope, + usage: usage_value, + last_value: init.call(instance, scope) || 0 + ) + end + rescue ActiveRecord::RecordNotUnique + lookup + end + end + + def up + duplicate_epic_ids = ApplicationRecord.connection.execute('SELECT iid, group_id, COUNT(*) FROM epics GROUP BY iid, group_id HAVING COUNT(*) > 1;') + + duplicate_epic_ids.each do |dup| + Epic.where(iid: dup['iid'], group_id: dup['group_id']).last(dup['count'] - 1).each do |epic| + new_iid = InternalId.generate_next(epic, + { namespace_id: epic.group_id }, + :epics, ->(instance, _) { instance.class.where(group_id: epic.group_id).maximum(:iid) } + ) + + epic.update!(iid: new_iid) + end + end + + add_concurrent_index :epics, [:group_id, :iid], unique: true, name: INDEX_NAME + end + + def down + # only remove the index, as we do not want to create the duplicates back + remove_concurrent_index :epics, [:group_id, :iid], name: INDEX_NAME + end +end |