diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-02-18 10:34:06 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-02-18 10:34:06 +0000 |
commit | 859a6fb938bb9ee2a317c46dfa4fcc1af49608f0 (patch) | |
tree | d7f2700abe6b4ffcb2dcfc80631b2d87d0609239 /db/post_migrate | |
parent | 446d496a6d000c73a304be52587cd9bbc7493136 (diff) | |
download | gitlab-ce-859a6fb938bb9ee2a317c46dfa4fcc1af49608f0.tar.gz |
Add latest changes from gitlab-org/gitlab@13-9-stable-eev13.9.0-rc42
Diffstat (limited to 'db/post_migrate')
20 files changed, 440 insertions, 4 deletions
diff --git a/db/post_migrate/20190214112022_schedule_sync_issuables_state_id.rb b/db/post_migrate/20190214112022_schedule_sync_issuables_state_id.rb index c05ad73d616..34536e22bbf 100644 --- a/db/post_migrate/20190214112022_schedule_sync_issuables_state_id.rb +++ b/db/post_migrate/20190214112022_schedule_sync_issuables_state_id.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class ScheduleSyncIssuablesStateId < ActiveRecord::Migration[5.0] # This migration schedules the sync of state_id for issues and merge requests # which are converting the state column from string to integer. diff --git a/db/post_migrate/20190606175050_encrypt_feature_flags_clients_tokens.rb b/db/post_migrate/20190606175050_encrypt_feature_flags_clients_tokens.rb index cb7d723670f..b8df41767f0 100644 --- a/db/post_migrate/20190606175050_encrypt_feature_flags_clients_tokens.rb +++ b/db/post_migrate/20190606175050_encrypt_feature_flags_clients_tokens.rb @@ -10,7 +10,7 @@ class EncryptFeatureFlagsClientsTokens < ActiveRecord::Migration[5.1] def up say_with_time("Encrypting tokens from operations_feature_flags_clients") do FeatureFlagsClient.where('token_encrypted is NULL AND token IS NOT NULL').find_each do |feature_flags_client| - token_encrypted = Gitlab::CryptoHelper.aes256_gcm_encrypt(feature_flags_client.token) + token_encrypted = Gitlab::CryptoHelper.aes256_gcm_encrypt(feature_flags_client.token, nonce: Gitlab::CryptoHelper::AES256_GCM_IV_STATIC) feature_flags_client.update!(token_encrypted: token_encrypted) end end diff --git a/db/post_migrate/20190711201818_encrypt_deploy_tokens_tokens.rb b/db/post_migrate/20190711201818_encrypt_deploy_tokens_tokens.rb index 2eb8d1ee11c..b2de98118b7 100644 --- a/db/post_migrate/20190711201818_encrypt_deploy_tokens_tokens.rb +++ b/db/post_migrate/20190711201818_encrypt_deploy_tokens_tokens.rb @@ -10,7 +10,7 @@ class EncryptDeployTokensTokens < ActiveRecord::Migration[5.1] def up say_with_time("Encrypting tokens from deploy_tokens") do DeploymentTokens.where('token_encrypted is NULL AND token IS NOT NULL').find_each(batch_size: 10000) do |deploy_token| - token_encrypted = Gitlab::CryptoHelper.aes256_gcm_encrypt(deploy_token.token) + token_encrypted = Gitlab::CryptoHelper.aes256_gcm_encrypt(deploy_token.token, nonce: Gitlab::CryptoHelper::AES256_GCM_IV_STATIC) deploy_token.update!(token_encrypted: token_encrypted) end end diff --git a/db/post_migrate/20200408132152_remove_namespaces_trial_ends_on.rb b/db/post_migrate/20200408132152_remove_namespaces_trial_ends_on.rb index cfe0daaf50f..a303b9d1869 100644 --- a/db/post_migrate/20200408132152_remove_namespaces_trial_ends_on.rb +++ b/db/post_migrate/20200408132152_remove_namespaces_trial_ends_on.rb @@ -18,7 +18,7 @@ class RemoveNamespacesTrialEndsOn < ActiveRecord::Migration[6.0] def down unless column_exists?(:namespaces, :trial_ends_on) with_lock_retries do - add_column :namespaces, :trial_ends_on, :datetime_with_timezone + add_column :namespaces, :trial_ends_on, :datetime_with_timezone # rubocop:disable Migration/AddColumnsToWideTables end end diff --git a/db/post_migrate/20200424043515_drop_namespaces_plan_id.rb b/db/post_migrate/20200424043515_drop_namespaces_plan_id.rb index 16a56b16e5a..b05f2da648a 100644 --- a/db/post_migrate/20200424043515_drop_namespaces_plan_id.rb +++ b/db/post_migrate/20200424043515_drop_namespaces_plan_id.rb @@ -16,7 +16,7 @@ class DropNamespacesPlanId < ActiveRecord::Migration[6.0] def down unless column_exists?(:namespaces, :plan_id) with_lock_retries do - add_column :namespaces, :plan_id, :integer + add_column :namespaces, :plan_id, :integer # rubocop:disable Migration/AddColumnsToWideTables end end diff --git a/db/post_migrate/20201030121314_schedule_update_existing_users_that_require_two_factor_auth.rb b/db/post_migrate/20201030121314_schedule_update_existing_users_that_require_two_factor_auth.rb index 2cb7c9c5250..6a031c28ed7 100644 --- a/db/post_migrate/20201030121314_schedule_update_existing_users_that_require_two_factor_auth.rb +++ b/db/post_migrate/20201030121314_schedule_update_existing_users_that_require_two_factor_auth.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # # frozen_string_literal: true class ScheduleUpdateExistingUsersThatRequireTwoFactorAuth < ActiveRecord::Migration[6.0] diff --git a/db/post_migrate/20201112130710_schedule_remove_duplicate_vulnerabilities_findings.rb b/db/post_migrate/20201112130710_schedule_remove_duplicate_vulnerabilities_findings.rb new file mode 100644 index 00000000000..d05516bd255 --- /dev/null +++ b/db/post_migrate/20201112130710_schedule_remove_duplicate_vulnerabilities_findings.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +class ScheduleRemoveDuplicateVulnerabilitiesFindings < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + INDEX_NAME = 'tmp_idx_deduplicate_vulnerability_occurrences' + + MIGRATION = 'RemoveDuplicateVulnerabilitiesFindings' + DELAY_INTERVAL = 2.minutes.to_i + BATCH_SIZE = 5_000 + + disable_ddl_transaction! + + class VulnerabilitiesFinding < ActiveRecord::Base + include ::EachBatch + self.table_name = "vulnerability_occurrences" + end + + def up + add_concurrent_index :vulnerability_occurrences, + %i[project_id report_type location_fingerprint primary_identifier_id id], + name: INDEX_NAME + + say "Scheduling #{MIGRATION} jobs" + queue_background_migration_jobs_by_range_at_intervals( + VulnerabilitiesFinding, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE + ) + end + + def down + remove_concurrent_index_by_name(:vulnerability_occurrences, INDEX_NAME) + end +end diff --git a/db/post_migrate/20201128210234_schedule_populate_issue_email_participants.rb b/db/post_migrate/20201128210234_schedule_populate_issue_email_participants.rb new file mode 100644 index 00000000000..6fea683944c --- /dev/null +++ b/db/post_migrate/20201128210234_schedule_populate_issue_email_participants.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class SchedulePopulateIssueEmailParticipants < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 1_000 + DELAY_INTERVAL = 2.minutes + MIGRATION = 'PopulateIssueEmailParticipants' + + disable_ddl_transaction! + + class Issue < ActiveRecord::Base + include EachBatch + + self.table_name = 'issues' + end + + def up + queue_background_migration_jobs_by_range_at_intervals( + Issue.where.not(service_desk_reply_to: nil), + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210105030125_cleanup_projects_with_bad_has_external_wiki_data.rb b/db/post_migrate/20210105030125_cleanup_projects_with_bad_has_external_wiki_data.rb new file mode 100644 index 00000000000..73725062bb3 --- /dev/null +++ b/db/post_migrate/20210105030125_cleanup_projects_with_bad_has_external_wiki_data.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +class CleanupProjectsWithBadHasExternalWikiData < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + TMP_INDEX_NAME = 'tmp_index_projects_on_id_where_has_external_wiki_is_true'.freeze + BATCH_SIZE = 100 + + disable_ddl_transaction! + + class Service < ActiveRecord::Base + include EachBatch + belongs_to :project + + self.table_name = 'services' + self.inheritance_column = :_type_disabled + end + + class Project < ActiveRecord::Base + include EachBatch + + self.table_name = 'projects' + end + + def up + update_projects_with_active_external_wikis + update_projects_without_active_external_wikis + end + + def down + # no-op : can't go back to incorrect data + end + + private + + def update_projects_with_active_external_wikis + # 11 projects are scoped in this query on GitLab.com. + scope = Service.where(active: true, type: 'ExternalWikiService').where.not(project_id: nil) + + scope.each_batch(of: BATCH_SIZE) do |relation| + scope_with_projects = relation + .joins(:project) + .select('project_id') + .merge(Project.where(has_external_wiki: false).where(pending_delete: false).where(archived: false)) + + execute(<<~SQL) + WITH project_ids_to_update (id) AS ( + #{scope_with_projects.to_sql} + ) + UPDATE projects SET has_external_wiki = true WHERE id IN (SELECT id FROM project_ids_to_update) + SQL + end + end + + def update_projects_without_active_external_wikis + # Add a temporary index to speed up the scoping of projects. + index_where = <<~SQL + ( + "projects"."has_external_wiki" = TRUE + ) + AND "projects"."pending_delete" = FALSE + AND "projects"."archived" = FALSE + SQL + + add_concurrent_index(:projects, :id, where: index_where, name: TMP_INDEX_NAME) + + services_sub_query = Service + .select('1') + .where('services.project_id = projects.id') + .where(type: 'ExternalWikiService') + .where(active: true) + + # 322 projects are scoped in this query on GitLab.com. + Project.where(index_where).each_batch(of: BATCH_SIZE) do |relation| + relation_with_exists_query = relation.where('NOT EXISTS (?)', services_sub_query) + execute(<<~SQL) + WITH project_ids_to_update (id) AS ( + #{relation_with_exists_query.select(:id).to_sql} + ) + UPDATE projects SET has_external_wiki = false WHERE id IN (SELECT id FROM project_ids_to_update) + SQL + end + + # Drop the temporary index. + remove_concurrent_index_by_name(:projects, TMP_INDEX_NAME) + end +end diff --git a/db/post_migrate/20210105052229_clean_up_asset_proxy_whitelist_rename_on_application_settings.rb b/db/post_migrate/20210105052229_clean_up_asset_proxy_whitelist_rename_on_application_settings.rb new file mode 100644 index 00000000000..8f6d312ae5e --- /dev/null +++ b/db/post_migrate/20210105052229_clean_up_asset_proxy_whitelist_rename_on_application_settings.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class CleanUpAssetProxyWhitelistRenameOnApplicationSettings < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers::V2 + + DOWNTIME = false + + disable_ddl_transaction! + + def up + cleanup_concurrent_column_rename :application_settings, + :asset_proxy_whitelist, + :asset_proxy_allowlist + end + + def down + undo_cleanup_concurrent_column_rename :application_settings, + :asset_proxy_whitelist, + :asset_proxy_allowlist + end +end diff --git a/db/post_migrate/20210111075105_schedule_uuid_population_for_security_findings.rb b/db/post_migrate/20210111075105_schedule_uuid_population_for_security_findings.rb new file mode 100644 index 00000000000..92241378495 --- /dev/null +++ b/db/post_migrate/20210111075105_schedule_uuid_population_for_security_findings.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class ScheduleUuidPopulationForSecurityFindings < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + MIGRATION_CLASS = 'PopulateUuidsForSecurityFindings' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 25 + + disable_ddl_transaction! + + def up + Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings.security_findings.each_batch(column: :scan_id, of: BATCH_SIZE) do |batch, index| + migrate_in( + DELAY_INTERVAL * index, + MIGRATION_CLASS, + batch.pluck(:scan_id) + ) + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210115220610_schedule_artifact_expiry_backfill.rb b/db/post_migrate/20210115220610_schedule_artifact_expiry_backfill.rb new file mode 100644 index 00000000000..44a76321495 --- /dev/null +++ b/db/post_migrate/20210115220610_schedule_artifact_expiry_backfill.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +class ScheduleArtifactExpiryBackfill < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + MIGRATION = 'BackfillArtifactExpiryDate'.freeze + SWITCH_DATE = Date.new(2020, 06, 22).freeze + INDEX_NAME = 'expired_artifacts_temp_index'.freeze + OLD_INDEX_CONDITION = "expire_at IS NULL AND created_at < '#{SWITCH_DATE}'" + INDEX_CONDITION = "expire_at IS NULL AND date(created_at AT TIME ZONE 'UTC') < '2020-06-22'::date".freeze + + disable_ddl_transaction! + + class JobArtifact < ActiveRecord::Base + include EachBatch + + self.table_name = 'ci_job_artifacts' + + scope :without_expiry_date, -> { where(expire_at: nil) } + scope :before_switch, -> { where("date(created_at AT TIME ZONE 'UTC') < ?::date", SWITCH_DATE) } + end + + def up + # Create temporary index for expired artifacts + # Needs to be removed in a later migration + remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME + add_concurrent_index(:ci_job_artifacts, %i(id created_at), where: INDEX_CONDITION, name: INDEX_NAME) + + queue_background_migration_jobs_by_range_at_intervals( + JobArtifact.without_expiry_date.before_switch, + MIGRATION, + 2.minutes, + batch_size: 200_000 + ) + end + + def down + remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME + add_concurrent_index(:ci_job_artifacts, %i(id created_at), where: OLD_INDEX_CONDITION, name: INDEX_NAME) + + Gitlab::BackgroundMigration.steal(MIGRATION) do |job| + job.delete + + false + end + end +end diff --git a/db/post_migrate/20210119122354_alter_vsa_issue_first_mentioned_in_commit_value.rb b/db/post_migrate/20210119122354_alter_vsa_issue_first_mentioned_in_commit_value.rb new file mode 100644 index 00000000000..132d72e180b --- /dev/null +++ b/db/post_migrate/20210119122354_alter_vsa_issue_first_mentioned_in_commit_value.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class AlterVsaIssueFirstMentionedInCommitValue < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + ISSUE_FIRST_MENTIONED_IN_COMMIT_FOSS = 2 + ISSUE_FIRST_MENTIONED_IN_COMMIT_EE = 6 + + class GroupStage < ActiveRecord::Base + self.table_name = 'analytics_cycle_analytics_group_stages' + + include EachBatch + end + + def up + GroupStage.each_batch(of: 100) do |relation| + relation + .where(start_event_identifier: ISSUE_FIRST_MENTIONED_IN_COMMIT_EE) + .update_all(start_event_identifier: ISSUE_FIRST_MENTIONED_IN_COMMIT_FOSS) + + relation + .where(end_event_identifier: ISSUE_FIRST_MENTIONED_IN_COMMIT_EE) + .update_all(end_event_identifier: ISSUE_FIRST_MENTIONED_IN_COMMIT_FOSS) + end + end + + def down + # rollback is not needed, the identifier "6" is the same as identifier "2" on the application level + end +end diff --git a/db/post_migrate/20210203143131_migrate_existing_devops_segments_to_groups.rb b/db/post_migrate/20210203143131_migrate_existing_devops_segments_to_groups.rb new file mode 100644 index 00000000000..5267e0fd658 --- /dev/null +++ b/db/post_migrate/20210203143131_migrate_existing_devops_segments_to_groups.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +# Data migration to migrate multi-selection segments into separate segments. +# Both tables involved are pretty-low traffic and the number +# of records in DB cannot exceed 400 +class MigrateExistingDevopsSegmentsToGroups < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups.new.perform + end + + def down + end +end diff --git a/db/post_migrate/20210205104425_add_new_post_eoa_plans.rb b/db/post_migrate/20210205104425_add_new_post_eoa_plans.rb new file mode 100644 index 00000000000..d1a5afbd314 --- /dev/null +++ b/db/post_migrate/20210205104425_add_new_post_eoa_plans.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class AddNewPostEoaPlans < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + execute "INSERT INTO plans (name, title, created_at, updated_at) VALUES ('premium', 'Premium (Formerly Silver)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" + execute "INSERT INTO plans (name, title, created_at, updated_at) VALUES ('ultimate', 'Ultimate (Formerly Gold)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" + end + + def down + execute "DELETE FROM plans WHERE name IN ('premium', 'ultimate')" + end +end diff --git a/db/post_migrate/20210205144537_remove_namespace_onboarding_actions_table.rb b/db/post_migrate/20210205144537_remove_namespace_onboarding_actions_table.rb new file mode 100644 index 00000000000..210b1d7822c --- /dev/null +++ b/db/post_migrate/20210205144537_remove_namespace_onboarding_actions_table.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class RemoveNamespaceOnboardingActionsTable < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + with_lock_retries do + drop_table :namespace_onboarding_actions + end + end + + def down + with_lock_retries do + create_table :namespace_onboarding_actions do |t| + t.references :namespace, index: true, null: false + t.datetime_with_timezone :created_at, null: false + t.integer :action, limit: 2, null: false + end + end + end +end diff --git a/db/post_migrate/20210205174154_remove_bad_dependency_proxy_manifests.rb b/db/post_migrate/20210205174154_remove_bad_dependency_proxy_manifests.rb new file mode 100644 index 00000000000..eb302fb7009 --- /dev/null +++ b/db/post_migrate/20210205174154_remove_bad_dependency_proxy_manifests.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class RemoveBadDependencyProxyManifests < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + # We run destroy on each record because we need the callback to remove + # the underlying files + DependencyProxy::Manifest.where.not(content_type: nil).destroy_all # rubocop:disable Cop/DestroyAll + end + + def down + # no op + end +end diff --git a/db/post_migrate/20210205213933_drop_alerts_service_data.rb b/db/post_migrate/20210205213933_drop_alerts_service_data.rb new file mode 100644 index 00000000000..6ad6c90290e --- /dev/null +++ b/db/post_migrate/20210205213933_drop_alerts_service_data.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class DropAlertsServiceData < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + with_lock_retries do + drop_table :alerts_service_data + end + end + + # rubocop:disable Migration/PreventStrings + def down + with_lock_retries do + create_table :alerts_service_data do |t| + t.bigint :service_id, null: false + t.timestamps_with_timezone + t.string :encrypted_token, limit: 255 + t.string :encrypted_token_iv, limit: 255 + end + end + end + # rubocop:enable Migration/PreventStrings +end diff --git a/db/post_migrate/20210205214003_remove_alerts_service_records_again.rb b/db/post_migrate/20210205214003_remove_alerts_service_records_again.rb new file mode 100644 index 00000000000..aabc6fc00cd --- /dev/null +++ b/db/post_migrate/20210205214003_remove_alerts_service_records_again.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class RemoveAlertsServiceRecordsAgain < ActiveRecord::Migration[6.0] + DOWNTIME = false + + disable_ddl_transaction! + + class Service < ActiveRecord::Base + self.table_name = 'services' + end + + def up + Service.delete_by(type: 'AlertsService') + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210210093901_backfill_updated_at_after_repository_storage_move.rb b/db/post_migrate/20210210093901_backfill_updated_at_after_repository_storage_move.rb new file mode 100644 index 00000000000..0631cc8095e --- /dev/null +++ b/db/post_migrate/20210210093901_backfill_updated_at_after_repository_storage_move.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class BackfillUpdatedAtAfterRepositoryStorageMove < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 10_000 + INTERVAL = 2.minutes + MIGRATION_CLASS = 'BackfillProjectUpdatedAtAfterRepositoryStorageMove' + + disable_ddl_transaction! + + class ProjectRepositoryStorageMove < ActiveRecord::Base + include EachBatch + + self.table_name = 'project_repository_storage_moves' + end + + def up + ProjectRepositoryStorageMove.reset_column_information + + ProjectRepositoryStorageMove.select(:project_id).distinct.each_batch(of: BATCH_SIZE, column: :project_id) do |batch, index| + migrate_in( + INTERVAL * index, + MIGRATION_CLASS, + batch.pluck(:project_id) + ) + end + end + + def down + # No-op + end +end |