From 41fe97390ceddf945f3d967b8fdb3de4c66b7dea Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 18 Mar 2022 20:02:30 +0000 Subject: Add latest changes from gitlab-org/gitlab@14-9-stable-ee --- .../20210329102724_add_new_trail_plans.rb | 4 +- ...al_plans_ci_daily_pipeline_schedule_triggers.rb | 4 +- ...3042_remove_duplicate_project_authorizations.rb | 111 +++++++++++++++++++++ .../20211026070408_backfill_issue_search_data.rb | 22 ++++ ..._delete_issue_merge_request_taggings_records.rb | 6 +- .../20211029102822_add_open_source_plan.rb | 4 +- ...ckfill_member_namespace_id_for_group_members.rb | 27 +++++ ...ex_job_artifacts_on_trace_type_and_expire_at.rb | 16 +++ ...20220131000001_schedule_trace_expiry_removal.rb | 55 ++++++++++ ...110725_backfill_cycle_analytics_aggregations.rb | 33 ++++++ ...0220204194347_encrypt_integration_properties.rb | 22 ++++ ...220207080758_update_api_indexes_for_projects.rb | 48 +++++++++ ...rsonal_namespace_project_maintainer_to_owner.rb | 25 +++++ ...onvert_stringified_raw_metadata_hash_to_json.rb | 27 +++++ ...ackage_files_limit_from_application_settings.rb | 18 ++++ ...ot_null_constraint_on_security_findings_uuid.rb | 13 +++ ...928_remove_show_diff_preview_in_email_column.rb | 13 +++ ..._null_constraint_for_security_scan_succeeded.rb | 11 ++ ...2524_create_not_null_constraint_releases_tag.rb | 13 +++ .../20220222192525_remove_null_releases.rb | 24 +++++ ...hedule_nullify_orphan_runner_id_on_ci_builds.rb | 27 +++++ ...0224000000_async_build_trace_expire_at_index.rb | 14 +++ ...ilds_on_name_and_id_parser_with_new_features.rb | 31 ++++++ ...225133705_cleanup_backfill_ci_queuing_tables.rb | 15 +++ ...220301093434_backfill_all_project_namespaces.rb | 30 ++++++ ...ilds_on_name_and_id_parser_with_new_features.rb | 28 ++++++ ...20220304165107_drop_partitioned_foreign_keys.rb | 19 ++++ ..._unique_index_on_security_training_providers.rb | 15 +++ ...220305223212_add_security_training_providers.rb | 40 ++++++++ ...ex_for_remove_duplicate_project_tag_releases.rb | 17 ++++ ...192610_remove_duplicate_project_tag_releases.rb | 26 +++++ ...ex_for_remove_duplicate_project_tag_releases.rb | 17 ++++ ...2725_create_unique_index_release_tag_project.rb | 23 +++++ ...kup_limit_setting_to_search_settings_cleanup.rb | 15 +++ ...ity_ci_builds_on_name_and_id_parser_features.rb | 26 +++++ ...i_runners_token_encrypted_values_on_projects.rb | 27 +++++ ...uplicate_ci_runners_token_values_on_projects.rb | 27 +++++ ...084838_remove_external_pull_request_tracking.rb | 15 +++ ...ove_leftover_external_pull_request_deletions.rb | 30 ++++++ ...9154855_add_index_on_issues_closed_incidents.rb | 15 +++ ...index_ci_job_artifacts_project_id_created_at.rb | 16 +++ ...roject_id_and_released_at_and_id_on_releases.rb | 16 +++ ...remove_dependency_list_usage_data_from_redis.rb | 13 +++ ...index_ci_job_artifacts_project_id_created_at.rb | 15 +++ 44 files changed, 1006 insertions(+), 7 deletions(-) create mode 100644 db/post_migrate/20210812013042_remove_duplicate_project_authorizations.rb create mode 100644 db/post_migrate/20211026070408_backfill_issue_search_data.rb create mode 100644 db/post_migrate/20220120211832_backfill_member_namespace_id_for_group_members.rb create mode 100644 db/post_migrate/20220131000000_index_job_artifacts_on_trace_type_and_expire_at.rb create mode 100644 db/post_migrate/20220131000001_schedule_trace_expiry_removal.rb create mode 100644 db/post_migrate/20220204110725_backfill_cycle_analytics_aggregations.rb create mode 100644 db/post_migrate/20220204194347_encrypt_integration_properties.rb create mode 100644 db/post_migrate/20220207080758_update_api_indexes_for_projects.rb create mode 100644 db/post_migrate/20220208080921_schedule_migrate_personal_namespace_project_maintainer_to_owner.rb create mode 100644 db/post_migrate/20220215190020_rerun_convert_stringified_raw_metadata_hash_to_json.rb create mode 100644 db/post_migrate/20220216201949_remove_package_files_limit_from_application_settings.rb create mode 100644 db/post_migrate/20220217135229_validate_not_null_constraint_on_security_findings_uuid.rb create mode 100644 db/post_migrate/20220221214928_remove_show_diff_preview_in_email_column.rb create mode 100644 db/post_migrate/20220222191845_remove_not_null_constraint_for_security_scan_succeeded.rb create mode 100644 db/post_migrate/20220222192524_create_not_null_constraint_releases_tag.rb create mode 100644 db/post_migrate/20220222192525_remove_null_releases.rb create mode 100644 db/post_migrate/20220223112304_schedule_nullify_orphan_runner_id_on_ci_builds.rb create mode 100644 db/post_migrate/20220224000000_async_build_trace_expire_at_index.rb create mode 100644 db/post_migrate/20220224204415_recreate_index_security_ci_builds_on_name_and_id_parser_with_new_features.rb create mode 100644 db/post_migrate/20220225133705_cleanup_backfill_ci_queuing_tables.rb create mode 100644 db/post_migrate/20220301093434_backfill_all_project_namespaces.rb create mode 100644 db/post_migrate/20220302203410_create_index_security_ci_builds_on_name_and_id_parser_with_new_features.rb create mode 100644 db/post_migrate/20220304165107_drop_partitioned_foreign_keys.rb create mode 100644 db/post_migrate/20220304201847_add_unique_index_on_security_training_providers.rb create mode 100644 db/post_migrate/20220305223212_add_security_training_providers.rb create mode 100644 db/post_migrate/20220307192534_create_index_for_remove_duplicate_project_tag_releases.rb create mode 100644 db/post_migrate/20220307192610_remove_duplicate_project_tag_releases.rb create mode 100644 db/post_migrate/20220307192645_remove_index_for_remove_duplicate_project_tag_releases.rb create mode 100644 db/post_migrate/20220307192725_create_unique_index_release_tag_project.rb create mode 100644 db/post_migrate/20220307203459_rename_user_email_lookup_limit_setting_to_search_settings_cleanup.rb create mode 100644 db/post_migrate/20220308000205_drop_old_index_security_ci_builds_on_name_and_id_parser_features.rb create mode 100644 db/post_migrate/20220308115219_schedule_reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb create mode 100644 db/post_migrate/20220308115502_schedule_reset_duplicate_ci_runners_token_values_on_projects.rb create mode 100644 db/post_migrate/20220309084838_remove_external_pull_request_tracking.rb create mode 100644 db/post_migrate/20220309084954_remove_leftover_external_pull_request_deletions.rb create mode 100644 db/post_migrate/20220309154855_add_index_on_issues_closed_incidents.rb create mode 100644 db/post_migrate/20220310095341_add_async_index_ci_job_artifacts_project_id_created_at.rb create mode 100644 db/post_migrate/20220310134207_add_index_project_id_and_released_at_and_id_on_releases.rb create mode 100644 db/post_migrate/20220310141349_remove_dependency_list_usage_data_from_redis.rb create mode 100644 db/post_migrate/20220314162342_add_index_ci_job_artifacts_project_id_created_at.rb (limited to 'db/post_migrate') diff --git a/db/post_migrate/20210329102724_add_new_trail_plans.rb b/db/post_migrate/20210329102724_add_new_trail_plans.rb index b142f6385f7..37c64bbd42d 100644 --- a/db/post_migrate/20210329102724_add_new_trail_plans.rb +++ b/db/post_migrate/20210329102724_add_new_trail_plans.rb @@ -24,7 +24,7 @@ class AddNewTrailPlans < ActiveRecord::Migration[6.0] end def up - return unless Gitlab.dev_env_or_com? + return unless Gitlab.com? ultimate_trial = Plan.create!(name: 'ultimate_trial', title: 'Ultimate Trial') premium_trial = Plan.create!(name: 'premium_trial', title: 'Premium Trial') @@ -34,7 +34,7 @@ class AddNewTrailPlans < ActiveRecord::Migration[6.0] end def down - return unless Gitlab.dev_env_or_com? + return unless Gitlab.com? Plan.where(name: %w(ultimate_trial premium_trial)).delete_all end diff --git a/db/post_migrate/20210805192450_update_trial_plans_ci_daily_pipeline_schedule_triggers.rb b/db/post_migrate/20210805192450_update_trial_plans_ci_daily_pipeline_schedule_triggers.rb index 9552058dd73..f27f61729a3 100644 --- a/db/post_migrate/20210805192450_update_trial_plans_ci_daily_pipeline_schedule_triggers.rb +++ b/db/post_migrate/20210805192450_update_trial_plans_ci_daily_pipeline_schedule_triggers.rb @@ -29,7 +29,7 @@ class UpdateTrialPlansCiDailyPipelineScheduleTriggers < ActiveRecord::Migration[ end def up - return unless Gitlab.dev_env_or_com? + return unless Gitlab.com? if plan_limits_present? create_or_update_plan_limit('ci_daily_pipeline_schedule_triggers', PREMIUM_TRIAL, EVERY_5_MINUTES) @@ -38,7 +38,7 @@ class UpdateTrialPlansCiDailyPipelineScheduleTriggers < ActiveRecord::Migration[ end def down - return unless Gitlab.dev_env_or_com? + return unless Gitlab.com? if plan_limits_present? create_or_update_plan_limit('ci_daily_pipeline_schedule_triggers', PREMIUM_TRIAL, 0) diff --git a/db/post_migrate/20210812013042_remove_duplicate_project_authorizations.rb b/db/post_migrate/20210812013042_remove_duplicate_project_authorizations.rb new file mode 100644 index 00000000000..6fdc30d09c6 --- /dev/null +++ b/db/post_migrate/20210812013042_remove_duplicate_project_authorizations.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +class RemoveDuplicateProjectAuthorizations < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + BATCH_SIZE = 10_000 + OLD_INDEX_NAME = 'index_project_authorizations_on_project_id_user_id' + INDEX_NAME = 'index_unique_project_authorizations_on_project_id_user_id' + + class ProjectAuthorization < ActiveRecord::Base + self.table_name = 'project_authorizations' + end + + disable_ddl_transaction! + + def up + batch do |first_record, last_record| + break if first_record.blank? + + # construct a range query where we filter records between the first and last records + rows = ActiveRecord::Base.connection.execute <<~SQL + SELECT user_id, project_id + FROM project_authorizations + WHERE + #{start_condition(first_record)} + #{end_condition(last_record)} + GROUP BY user_id, project_id + HAVING COUNT(*) > 1 + SQL + + rows.each do |row| + deduplicate_item(row['project_id'], row['user_id']) + end + end + + add_concurrent_index :project_authorizations, [:project_id, :user_id], unique: true, name: INDEX_NAME + remove_concurrent_index_by_name :project_authorizations, OLD_INDEX_NAME + end + + def down + add_concurrent_index(:project_authorizations, [:project_id, :user_id], name: OLD_INDEX_NAME) + remove_concurrent_index_by_name(:project_authorizations, INDEX_NAME) + end + + private + + def start_condition(record) + "(user_id, project_id) >= (#{Integer(record.user_id)}, #{Integer(record.project_id)})" + end + + def end_condition(record) + return "" unless record + + "AND (user_id, project_id) <= (#{Integer(record.user_id)}, #{Integer(record.project_id)})" + end + + def batch(&block) + order = Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'user_id', + order_expression: ProjectAuthorization.arel_table[:user_id].asc, + nullable: :not_nullable, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'project_id', + order_expression: ProjectAuthorization.arel_table[:project_id].asc, + nullable: :not_nullable, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'access_level', + order_expression: ProjectAuthorization.arel_table[:access_level].asc, + nullable: :not_nullable, + distinct: true + ) + ]) + + scope = ProjectAuthorization.order(order) + cursor = {} + loop do + current_scope = scope.dup + + relation = order.apply_cursor_conditions(current_scope, cursor) + first_record = relation.take + last_record = relation.offset(BATCH_SIZE).take + + yield first_record, last_record + + break if last_record.blank? + + cursor = order.cursor_attributes_for_node(last_record) + end + end + + def deduplicate_item(project_id, user_id) + auth_records = ProjectAuthorization.where(project_id: project_id, user_id: user_id).order(access_level: :desc).to_a + + ActiveRecord::Base.transaction do + # Keep the highest access level and destroy the rest. + auth_records[1..].each do |record| + ProjectAuthorization + .where( + project_id: record.project_id, + user_id: record.user_id, + access_level: record.access_level + ).delete_all + end + end + end +end diff --git a/db/post_migrate/20211026070408_backfill_issue_search_data.rb b/db/post_migrate/20211026070408_backfill_issue_search_data.rb new file mode 100644 index 00000000000..a840adcb991 --- /dev/null +++ b/db/post_migrate/20211026070408_backfill_issue_search_data.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class BackfillIssueSearchData < Gitlab::Database::Migration[1.0] + MIGRATION = 'BackfillIssueSearchData' + + def up + queue_batched_background_migration( + MIGRATION, + :issues, + :id, + batch_size: 100_000, + sub_batch_size: 1_000, + job_interval: 5.minutes + ) + end + + def down + Gitlab::Database::BackgroundMigration::BatchedMigration + .for_configuration(MIGRATION, :issues, :id, []) + .delete_all + end +end diff --git a/db/post_migrate/20211028100843_delete_issue_merge_request_taggings_records.rb b/db/post_migrate/20211028100843_delete_issue_merge_request_taggings_records.rb index 1ceaa1b5aef..a53f5a4fc25 100644 --- a/db/post_migrate/20211028100843_delete_issue_merge_request_taggings_records.rb +++ b/db/post_migrate/20211028100843_delete_issue_merge_request_taggings_records.rb @@ -8,8 +8,12 @@ class DeleteIssueMergeRequestTaggingsRecords < Gitlab::Database::Migration[1.0] BATCH_SIZE = 3_000 TAGGABLE_TYPES = %w(Issue MergeRequest) + class Tagging < ActiveRecord::Base + self.table_name = "taggings" + end + def up - sleep 2 while ActsAsTaggableOn::Tagging.where(taggable_type: TAGGABLE_TYPES).limit(BATCH_SIZE).delete_all > 0 + sleep 2 while Tagging.where(taggable_type: TAGGABLE_TYPES).limit(BATCH_SIZE).delete_all > 0 remove_concurrent_index_by_name :taggings, INDEX_NAME end diff --git a/db/post_migrate/20211029102822_add_open_source_plan.rb b/db/post_migrate/20211029102822_add_open_source_plan.rb index 00266640f03..bb65637ffca 100644 --- a/db/post_migrate/20211029102822_add_open_source_plan.rb +++ b/db/post_migrate/20211029102822_add_open_source_plan.rb @@ -24,7 +24,7 @@ class AddOpenSourcePlan < Gitlab::Database::Migration[1.0] end def up - return unless Gitlab.dev_env_or_com? + return unless Gitlab.com? opensource = Plan.create!(name: 'opensource', title: 'Open Source Program') @@ -32,7 +32,7 @@ class AddOpenSourcePlan < Gitlab::Database::Migration[1.0] end def down - return unless Gitlab.dev_env_or_com? + return unless Gitlab.com? Plan.where(name: 'opensource').delete_all end diff --git a/db/post_migrate/20220120211832_backfill_member_namespace_id_for_group_members.rb b/db/post_migrate/20220120211832_backfill_member_namespace_id_for_group_members.rb new file mode 100644 index 00000000000..947c0a1edd0 --- /dev/null +++ b/db/post_migrate/20220120211832_backfill_member_namespace_id_for_group_members.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class BackfillMemberNamespaceIdForGroupMembers < Gitlab::Database::Migration[1.0] + MIGRATION = 'BackfillMemberNamespaceForGroupMembers' + INTERVAL = 2.minutes + BATCH_SIZE = 1_000 + MAX_BATCH_SIZE = 2_000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :members, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + Gitlab::Database::BackgroundMigration::BatchedMigration + .for_configuration(MIGRATION, :members, :id, []) + .delete_all + end +end diff --git a/db/post_migrate/20220131000000_index_job_artifacts_on_trace_type_and_expire_at.rb b/db/post_migrate/20220131000000_index_job_artifacts_on_trace_type_and_expire_at.rb new file mode 100644 index 00000000000..f47c04dd2ef --- /dev/null +++ b/db/post_migrate/20220131000000_index_job_artifacts_on_trace_type_and_expire_at.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexJobArtifactsOnTraceTypeAndExpireAt < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + INDEX_NAME = 'tmp_index_ci_job_artifacts_on_id_where_trace_and_expire_at' + TIMESTAMPS = "'2021-04-22 00:00:00', '2021-05-22 00:00:00', '2021-06-22 00:00:00', '2022-01-22 00:00:00', '2022-02-22 00:00:00', '2022-03-22 00:00:00', '2022-04-22 00:00:00'" + + def up + add_concurrent_index :ci_job_artifacts, :id, where: "file_type = 3 AND expire_at IN (#{TIMESTAMPS})", name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME + end +end diff --git a/db/post_migrate/20220131000001_schedule_trace_expiry_removal.rb b/db/post_migrate/20220131000001_schedule_trace_expiry_removal.rb new file mode 100644 index 00000000000..8e282a9b8c2 --- /dev/null +++ b/db/post_migrate/20220131000001_schedule_trace_expiry_removal.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +class ScheduleTraceExpiryRemoval < Gitlab::Database::Migration[1.0] + MIGRATION = 'RemoveAllTraceExpirationDates' + BATCH_SIZE = 100_000 + DELAY_INTERVAL = 4.minutes + + disable_ddl_transaction! + + # Stubbed class to connect to the CI database + # connects_to has to be called in abstract classes. + class MultiDbAdaptableClass < ActiveRecord::Base + self.abstract_class = true + + if Gitlab::Database.has_config?(:ci) + connects_to database: { writing: :ci, reading: :ci } + end + end + + # Stubbed class to access the ci_job_artifacts table + class JobArtifact < MultiDbAdaptableClass + include EachBatch + + self.table_name = 'ci_job_artifacts' + + TARGET_TIMESTAMPS = [ + Date.new(2021, 04, 22).midnight.utc, + Date.new(2021, 05, 22).midnight.utc, + Date.new(2021, 06, 22).midnight.utc, + Date.new(2022, 01, 22).midnight.utc, + Date.new(2022, 02, 22).midnight.utc, + Date.new(2022, 03, 22).midnight.utc, + Date.new(2022, 04, 22).midnight.utc + ].freeze + + scope :in_targeted_timestamps, -> { where(expire_at: TARGET_TIMESTAMPS) } + scope :traces, -> { where(file_type: 3) } + end + + def up + return unless Gitlab.com? + + queue_background_migration_jobs_by_range_at_intervals( + JobArtifact.traces.in_targeted_timestamps, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + track_jobs: true + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220204110725_backfill_cycle_analytics_aggregations.rb b/db/post_migrate/20220204110725_backfill_cycle_analytics_aggregations.rb new file mode 100644 index 00000000000..933ad747c5c --- /dev/null +++ b/db/post_migrate/20220204110725_backfill_cycle_analytics_aggregations.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class BackfillCycleAnalyticsAggregations < Gitlab::Database::Migration[1.0] + BATCH_SIZE = 50 + + def up + model = define_batchable_model('analytics_cycle_analytics_group_value_streams') + + model.each_batch(of: BATCH_SIZE) do |relation| + execute <<~SQL + WITH records_to_be_inserted AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + SELECT root_ancestor.id AS group_id + FROM (#{relation.select(:group_id).to_sql}) as value_streams, + LATERAL ( + WITH RECURSIVE "base_and_ancestors" AS ( + (SELECT "namespaces"."id", "namespaces"."parent_id" FROM "namespaces" WHERE "namespaces"."id" = value_streams.group_id) + UNION + (SELECT "namespaces"."id", "namespaces"."parent_id" FROM "namespaces", "base_and_ancestors" WHERE "namespaces"."id" = "base_and_ancestors"."parent_id") + ) + SELECT "namespaces"."id" FROM "base_and_ancestors" as "namespaces" WHERE parent_id IS NULL LIMIT 1 + ) as root_ancestor + ) + INSERT INTO "analytics_cycle_analytics_aggregations" + SELECT * FROM "records_to_be_inserted" + ON CONFLICT DO NOTHING + SQL + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220204194347_encrypt_integration_properties.rb b/db/post_migrate/20220204194347_encrypt_integration_properties.rb new file mode 100644 index 00000000000..82dd3a05e1d --- /dev/null +++ b/db/post_migrate/20220204194347_encrypt_integration_properties.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class EncryptIntegrationProperties < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + MIGRATION = 'EncryptIntegrationProperties' + BATCH_SIZE = 1_000 + INTERVAL = 2.minutes.to_i + + def up + queue_background_migration_jobs_by_range_at_intervals( + define_batchable_model('integrations').all, + MIGRATION, + INTERVAL, + track_jobs: true, + batch_size: BATCH_SIZE + ) + end + + def down + # this migration is not reversible + end +end diff --git a/db/post_migrate/20220207080758_update_api_indexes_for_projects.rb b/db/post_migrate/20220207080758_update_api_indexes_for_projects.rb new file mode 100644 index 00000000000..3275912b0ab --- /dev/null +++ b/db/post_migrate/20220207080758_update_api_indexes_for_projects.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +class UpdateApiIndexesForProjects < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + ARCHIVED_INDEX_NAME = 'idx_projects_api_created_at_id_for_archived' + OLD_ARCHIVED_INDEX_NAME = 'index_projects_api_created_at_id_for_archived' + PUBLIC_AND_ARCHIVED_INDEX_NAME = 'idx_projects_api_created_at_id_for_archived_vis20' + OLD_PUBLIC_AND_ARCHIVED_INDEX_NAME = 'index_projects_api_created_at_id_for_archived_vis20' + INTERNAL_PROJECTS_INDEX_NAME = 'idx_projects_api_created_at_id_for_vis10' + OLD_INTERNAL_PROJECTS_INDEX_NAME = 'index_projects_api_created_at_id_for_vis10' + + def up + add_concurrent_index :projects, [:created_at, :id], + where: "archived = true AND pending_delete = false AND hidden = false", + name: ARCHIVED_INDEX_NAME + + add_concurrent_index :projects, [:created_at, :id], + where: "archived = true AND visibility_level = 20 AND pending_delete = false AND hidden = false", + name: PUBLIC_AND_ARCHIVED_INDEX_NAME + + add_concurrent_index :projects, [:created_at, :id], + where: "visibility_level = 10 AND pending_delete = false AND hidden = false", + name: INTERNAL_PROJECTS_INDEX_NAME + + remove_concurrent_index_by_name :projects, OLD_ARCHIVED_INDEX_NAME + remove_concurrent_index_by_name :projects, OLD_PUBLIC_AND_ARCHIVED_INDEX_NAME + remove_concurrent_index_by_name :projects, OLD_INTERNAL_PROJECTS_INDEX_NAME + end + + def down + add_concurrent_index :projects, [:created_at, :id], + where: "archived = true AND pending_delete = false", + name: OLD_ARCHIVED_INDEX_NAME + + add_concurrent_index :projects, [:created_at, :id], + where: "archived = true AND visibility_level = 20 AND pending_delete = false", + name: OLD_PUBLIC_AND_ARCHIVED_INDEX_NAME + + add_concurrent_index :projects, [:created_at, :id], + where: "visibility_level = 10 AND pending_delete = false", + name: OLD_INTERNAL_PROJECTS_INDEX_NAME + + remove_concurrent_index_by_name :projects, ARCHIVED_INDEX_NAME + remove_concurrent_index_by_name :projects, PUBLIC_AND_ARCHIVED_INDEX_NAME + remove_concurrent_index_by_name :projects, INTERNAL_PROJECTS_INDEX_NAME + end +end diff --git a/db/post_migrate/20220208080921_schedule_migrate_personal_namespace_project_maintainer_to_owner.rb b/db/post_migrate/20220208080921_schedule_migrate_personal_namespace_project_maintainer_to_owner.rb new file mode 100644 index 00000000000..633570aeaa0 --- /dev/null +++ b/db/post_migrate/20220208080921_schedule_migrate_personal_namespace_project_maintainer_to_owner.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class ScheduleMigratePersonalNamespaceProjectMaintainerToOwner < Gitlab::Database::Migration[1.0] + MIGRATION = 'MigratePersonalNamespaceProjectMaintainerToOwner' + INTERVAL = 2.minutes + BATCH_SIZE = 1_000 + SUB_BATCH_SIZE = 200 + + disable_ddl_transaction! + + def up + queue_batched_background_migration( + MIGRATION, + :members, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220215190020_rerun_convert_stringified_raw_metadata_hash_to_json.rb b/db/post_migrate/20220215190020_rerun_convert_stringified_raw_metadata_hash_to_json.rb new file mode 100644 index 00000000000..1f36132c578 --- /dev/null +++ b/db/post_migrate/20220215190020_rerun_convert_stringified_raw_metadata_hash_to_json.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class RerunConvertStringifiedRawMetadataHashToJson < Gitlab::Database::Migration[1.0] + MIGRATION_CLASS = Gitlab::BackgroundMigration::FixVulnerabilityOccurrencesWithHashesAsRawMetadata + MODEL_CLASS = MIGRATION_CLASS::Finding + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 500 + + disable_ddl_transaction! + + def up + queue_background_migration_jobs_by_range_at_intervals( + MODEL_CLASS.by_api_report_types, + MIGRATION_CLASS.name.demodulize, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + track_jobs: true + ) + end + + def down + # no-op + + # up fixes invalid data by updating columns in-place. + # It is a backwards-compatible change, and reversing it in a downgrade would not be desirable. + end +end diff --git a/db/post_migrate/20220216201949_remove_package_files_limit_from_application_settings.rb b/db/post_migrate/20220216201949_remove_package_files_limit_from_application_settings.rb new file mode 100644 index 00000000000..589b75dd918 --- /dev/null +++ b/db/post_migrate/20220216201949_remove_package_files_limit_from_application_settings.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class RemovePackageFilesLimitFromApplicationSettings < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + def up + return unless column_exists?(:application_settings, :max_package_files_for_package_destruction) + + remove_column :application_settings, :max_package_files_for_package_destruction, :smallint + end + + def down + add_column :application_settings, :max_package_files_for_package_destruction, :smallint, default: 100, null: false + add_check_constraint :application_settings, + 'max_package_files_for_package_destruction > 0', + 'app_settings_max_package_files_for_package_destruction_positive' + end +end diff --git a/db/post_migrate/20220217135229_validate_not_null_constraint_on_security_findings_uuid.rb b/db/post_migrate/20220217135229_validate_not_null_constraint_on_security_findings_uuid.rb new file mode 100644 index 00000000000..9cc04cef757 --- /dev/null +++ b/db/post_migrate/20220217135229_validate_not_null_constraint_on_security_findings_uuid.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class ValidateNotNullConstraintOnSecurityFindingsUuid < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + def up + validate_not_null_constraint(:security_findings, :uuid) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220221214928_remove_show_diff_preview_in_email_column.rb b/db/post_migrate/20220221214928_remove_show_diff_preview_in_email_column.rb new file mode 100644 index 00000000000..41cd3446c7a --- /dev/null +++ b/db/post_migrate/20220221214928_remove_show_diff_preview_in_email_column.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class RemoveShowDiffPreviewInEmailColumn < Gitlab::Database::Migration[1.0] + enable_lock_retries! + + def up + remove_column :project_settings, :show_diff_preview_in_email, :boolean + end + + def down + add_column :project_settings, :show_diff_preview_in_email, :boolean, default: true, null: false + end +end diff --git a/db/post_migrate/20220222191845_remove_not_null_constraint_for_security_scan_succeeded.rb b/db/post_migrate/20220222191845_remove_not_null_constraint_for_security_scan_succeeded.rb new file mode 100644 index 00000000000..4d4a20b83b3 --- /dev/null +++ b/db/post_migrate/20220222191845_remove_not_null_constraint_for_security_scan_succeeded.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class RemoveNotNullConstraintForSecurityScanSucceeded < Gitlab::Database::Migration[1.0] + def up + change_column_null :analytics_devops_adoption_snapshots, :security_scan_succeeded, true + end + + def down + # There may now be nulls in the table, so we cannot re-add the constraint here. + end +end diff --git a/db/post_migrate/20220222192524_create_not_null_constraint_releases_tag.rb b/db/post_migrate/20220222192524_create_not_null_constraint_releases_tag.rb new file mode 100644 index 00000000000..2bb5ba18743 --- /dev/null +++ b/db/post_migrate/20220222192524_create_not_null_constraint_releases_tag.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class CreateNotNullConstraintReleasesTag < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + def up + add_not_null_constraint :releases, :tag, constraint_name: 'releases_not_null_tag', validate: false + end + + def down + remove_not_null_constraint :releases, :tag, constraint_name: 'releases_not_null_tag' + end +end diff --git a/db/post_migrate/20220222192525_remove_null_releases.rb b/db/post_migrate/20220222192525_remove_null_releases.rb new file mode 100644 index 00000000000..4efd5393122 --- /dev/null +++ b/db/post_migrate/20220222192525_remove_null_releases.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class RemoveNullReleases < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + class Release < ActiveRecord::Base + include EachBatch + + self.table_name = 'releases' + end + + def up + Release.all.each_batch(of: 25000) do |rel| + rel.where(tag: nil).delete_all + end + end + + def down + # no-op + # + # releases with the same tag within a project have been removed + # and therefore the duplicate release data is no longer available + end +end diff --git a/db/post_migrate/20220223112304_schedule_nullify_orphan_runner_id_on_ci_builds.rb b/db/post_migrate/20220223112304_schedule_nullify_orphan_runner_id_on_ci_builds.rb new file mode 100644 index 00000000000..e4005885c3b --- /dev/null +++ b/db/post_migrate/20220223112304_schedule_nullify_orphan_runner_id_on_ci_builds.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class ScheduleNullifyOrphanRunnerIdOnCiBuilds < Gitlab::Database::Migration[1.0] + MIGRATION = 'NullifyOrphanRunnerIdOnCiBuilds' + INTERVAL = 2.minutes + BATCH_SIZE = 50_000 + MAX_BATCH_SIZE = 150_000 + SUB_BATCH_SIZE = 500 + + def up + queue_batched_background_migration( + MIGRATION, + :ci_builds, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + Gitlab::Database::BackgroundMigration::BatchedMigration + .for_configuration(MIGRATION, :ci_builds, :id, []) + .delete_all + end +end diff --git a/db/post_migrate/20220224000000_async_build_trace_expire_at_index.rb b/db/post_migrate/20220224000000_async_build_trace_expire_at_index.rb new file mode 100644 index 00000000000..b22f3e7996f --- /dev/null +++ b/db/post_migrate/20220224000000_async_build_trace_expire_at_index.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class AsyncBuildTraceExpireAtIndex < Gitlab::Database::Migration[1.0] + INDEX_NAME = 'tmp_index_ci_job_artifacts_on_id_where_trace_and_expire_at' + TIMESTAMPS = "'2021-04-22 00:00:00', '2021-05-22 00:00:00', '2021-06-22 00:00:00', '2022-01-22 00:00:00', '2022-02-22 00:00:00', '2022-03-22 00:00:00', '2022-04-22 00:00:00'" + + def up + prepare_async_index :ci_job_artifacts, :id, where: "file_type = 3 AND expire_at IN (#{TIMESTAMPS})", name: INDEX_NAME + end + + def down + unprepare_async_index :ci_builds, :id, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20220224204415_recreate_index_security_ci_builds_on_name_and_id_parser_with_new_features.rb b/db/post_migrate/20220224204415_recreate_index_security_ci_builds_on_name_and_id_parser_with_new_features.rb new file mode 100644 index 00000000000..feb0f2c83ab --- /dev/null +++ b/db/post_migrate/20220224204415_recreate_index_security_ci_builds_on_name_and_id_parser_with_new_features.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class RecreateIndexSecurityCiBuildsOnNameAndIdParserWithNewFeatures < Gitlab::Database::Migration[1.0] + TABLE = "ci_builds" + OLD_INDEX_NAME = "index_security_ci_builds_on_name_and_id_parser_features" + NEW_INDEX_NAME = "index_security_ci_builds_on_name_and_id_parser_features_old" + COLUMNS = %i[name id] + CONSTRAINTS = "(name::text = ANY (ARRAY['container_scanning'::character varying::text, + 'dast'::character varying::text, + 'dependency_scanning'::character varying::text, + 'license_management'::character varying::text, + 'sast'::character varying::text, + 'secret_detection'::character varying::text, + 'coverage_fuzzing'::character varying::text, + 'license_scanning'::character varying::text, + 'apifuzzer_fuzz'::character varying::text, + 'apifuzzer_fuzz_dnd'::character varying::text]) + ) AND type::text = 'Ci::Build'::text" + + enable_lock_retries! + + def up + rename_index(TABLE, OLD_INDEX_NAME, NEW_INDEX_NAME) + prepare_async_index TABLE, COLUMNS, name: OLD_INDEX_NAME, where: CONSTRAINTS + end + + def down + unprepare_async_index TABLE, COLUMNS, name: OLD_INDEX_NAME + rename_index(TABLE, NEW_INDEX_NAME, OLD_INDEX_NAME) + end +end diff --git a/db/post_migrate/20220225133705_cleanup_backfill_ci_queuing_tables.rb b/db/post_migrate/20220225133705_cleanup_backfill_ci_queuing_tables.rb new file mode 100644 index 00000000000..4fcf8a28727 --- /dev/null +++ b/db/post_migrate/20220225133705_cleanup_backfill_ci_queuing_tables.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class CleanupBackfillCiQueuingTables < Gitlab::Database::Migration[1.0] + MIGRATION = 'BackfillCiQueuingTables' + + disable_ddl_transaction! + + def up + finalize_background_migration(MIGRATION) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220301093434_backfill_all_project_namespaces.rb b/db/post_migrate/20220301093434_backfill_all_project_namespaces.rb new file mode 100644 index 00000000000..7071e6241ce --- /dev/null +++ b/db/post_migrate/20220301093434_backfill_all_project_namespaces.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class BackfillAllProjectNamespaces < Gitlab::Database::Migration[1.0] + MIGRATION = 'ProjectNamespaces::BackfillProjectNamespaces' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1_000 + MAX_BATCH_SIZE = 5_000 + SUB_BATCH_SIZE = 10 + + disable_ddl_transaction! + + def up + queue_batched_background_migration( + MIGRATION, + :projects, + :id, + nil, + 'up', + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + Gitlab::Database::BackgroundMigration::BatchedMigration + .for_configuration(MIGRATION, :projects, :id, [nil, 'up']).delete_all + end +end diff --git a/db/post_migrate/20220302203410_create_index_security_ci_builds_on_name_and_id_parser_with_new_features.rb b/db/post_migrate/20220302203410_create_index_security_ci_builds_on_name_and_id_parser_with_new_features.rb new file mode 100644 index 00000000000..5be6bb00269 --- /dev/null +++ b/db/post_migrate/20220302203410_create_index_security_ci_builds_on_name_and_id_parser_with_new_features.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class CreateIndexSecurityCiBuildsOnNameAndIdParserWithNewFeatures < Gitlab::Database::Migration[1.0] + TABLE = "ci_builds" + COLUMNS = %i[name id] + INDEX_NAME = "index_security_ci_builds_on_name_and_id_parser_features" + CONSTRAINTS = "(name::text = ANY (ARRAY['container_scanning'::character varying::text, + 'dast'::character varying::text, + 'dependency_scanning'::character varying::text, + 'license_management'::character varying::text, + 'sast'::character varying::text, + 'secret_detection'::character varying::text, + 'coverage_fuzzing'::character varying::text, + 'license_scanning'::character varying::text, + 'apifuzzer_fuzz'::character varying::text, + 'apifuzzer_fuzz_dnd'::character varying::text]) + ) AND type::text = 'Ci::Build'::text" + + disable_ddl_transaction! + + def up + add_concurrent_index(TABLE, COLUMNS, name: INDEX_NAME, where: CONSTRAINTS) + end + + def down + remove_concurrent_index(TABLE, COLUMNS, name: INDEX_NAME, where: CONSTRAINTS) + end +end diff --git a/db/post_migrate/20220304165107_drop_partitioned_foreign_keys.rb b/db/post_migrate/20220304165107_drop_partitioned_foreign_keys.rb new file mode 100644 index 00000000000..43f89b05fa4 --- /dev/null +++ b/db/post_migrate/20220304165107_drop_partitioned_foreign_keys.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class DropPartitionedForeignKeys < Gitlab::Database::Migration[1.0] + def up + drop_table :partitioned_foreign_keys + end + + def down + create_table :partitioned_foreign_keys do |t| + t.boolean :cascade_delete, null: false, default: true + t.text :from_table, null: false, limit: 63 + t.text :from_column, null: false, limit: 63 + t.text :to_table, null: false, limit: 63 + t.text :to_column, null: false, limit: 63 + + t.index [:to_table, :from_table, :from_column], unique: true, name: :index_partitioned_foreign_keys_unique_index + end + end +end diff --git a/db/post_migrate/20220304201847_add_unique_index_on_security_training_providers.rb b/db/post_migrate/20220304201847_add_unique_index_on_security_training_providers.rb new file mode 100644 index 00000000000..e78b8fd48ca --- /dev/null +++ b/db/post_migrate/20220304201847_add_unique_index_on_security_training_providers.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddUniqueIndexOnSecurityTrainingProviders < Gitlab::Database::Migration[1.0] + INDEX_NAME = 'index_security_training_providers_on_unique_name' + + disable_ddl_transaction! + + def up + add_concurrent_index :security_training_providers, :name, unique: true, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :security_training_providers, INDEX_NAME + end +end diff --git a/db/post_migrate/20220305223212_add_security_training_providers.rb b/db/post_migrate/20220305223212_add_security_training_providers.rb new file mode 100644 index 00000000000..fbddee0ae99 --- /dev/null +++ b/db/post_migrate/20220305223212_add_security_training_providers.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class AddSecurityTrainingProviders < Gitlab::Database::Migration[1.0] + KONTRA_DATA = { + name: 'Kontra', + description: "Kontra Application Security provides interactive developer security education that + enables engineers to quickly learn security best practices + and fix issues in their code by analysing real-world software security vulnerabilities.", + url: "https://application.security/api/webhook/gitlab/exercises/search" + } + + SCW_DATA = { + name: 'Secure Code Warrior', + description: "Resolve vulnerabilities faster and confidently with highly relevant and bite-sized secure coding learning.", + url: "https://integration-api.securecodewarrior.com/api/v1/trial" + } + + module Security + class TrainingProvider < ActiveRecord::Base + self.table_name = 'security_training_providers' + end + end + + def up + current_time = Time.current + timestamps = { created_at: current_time, updated_at: current_time } + + Security::TrainingProvider.reset_column_information + + # upsert providers + Security::TrainingProvider.upsert_all([KONTRA_DATA.merge(timestamps), SCW_DATA.merge(timestamps)]) + end + + def down + Security::TrainingProvider.reset_column_information + + Security::TrainingProvider.find_by(name: KONTRA_DATA[:name])&.destroy + Security::TrainingProvider.find_by(name: SCW_DATA[:name])&.destroy + end +end diff --git a/db/post_migrate/20220307192534_create_index_for_remove_duplicate_project_tag_releases.rb b/db/post_migrate/20220307192534_create_index_for_remove_duplicate_project_tag_releases.rb new file mode 100644 index 00000000000..3e580c013c3 --- /dev/null +++ b/db/post_migrate/20220307192534_create_index_for_remove_duplicate_project_tag_releases.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class CreateIndexForRemoveDuplicateProjectTagReleases < Gitlab::Database::Migration[1.0] + INDEX_NAME = 'index_releases_on_id_project_id_tag' + + disable_ddl_transaction! + + def up + add_concurrent_index :releases, + %i[project_id tag id], + name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :releases, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20220307192610_remove_duplicate_project_tag_releases.rb b/db/post_migrate/20220307192610_remove_duplicate_project_tag_releases.rb new file mode 100644 index 00000000000..d8b99380825 --- /dev/null +++ b/db/post_migrate/20220307192610_remove_duplicate_project_tag_releases.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class RemoveDuplicateProjectTagReleases < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + class Release < ActiveRecord::Base + include EachBatch + + self.table_name = 'releases' + end + + def up + Release.each_batch(of: 5000) do |relation| + relation + .where('exists (select 1 from releases r2 where r2.project_id = releases.project_id and r2.tag = releases.tag and r2.id > releases.id)') + .delete_all + end + end + + def down + # no-op + # + # releases with the same tag within a project have been removed + # and therefore the duplicate release data is no longer available + end +end diff --git a/db/post_migrate/20220307192645_remove_index_for_remove_duplicate_project_tag_releases.rb b/db/post_migrate/20220307192645_remove_index_for_remove_duplicate_project_tag_releases.rb new file mode 100644 index 00000000000..8efb24daff9 --- /dev/null +++ b/db/post_migrate/20220307192645_remove_index_for_remove_duplicate_project_tag_releases.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class RemoveIndexForRemoveDuplicateProjectTagReleases < Gitlab::Database::Migration[1.0] + INDEX_NAME = 'index_releases_on_id_project_id_tag' + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name :releases, name: INDEX_NAME + end + + def down + add_concurrent_index :releases, + %i[project_id tag id], + name: INDEX_NAME + end +end diff --git a/db/post_migrate/20220307192725_create_unique_index_release_tag_project.rb b/db/post_migrate/20220307192725_create_unique_index_release_tag_project.rb new file mode 100644 index 00000000000..8540f19f079 --- /dev/null +++ b/db/post_migrate/20220307192725_create_unique_index_release_tag_project.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class CreateUniqueIndexReleaseTagProject < Gitlab::Database::Migration[1.0] + INDEX_NAME = 'index_releases_on_project_tag_unique' + OLD_INDEX_NAME = 'index_releases_on_project_id_and_tag' + + disable_ddl_transaction! + + def up + add_concurrent_index :releases, + %i[project_id tag], + unique: true, + name: INDEX_NAME + remove_concurrent_index_by_name :releases, name: OLD_INDEX_NAME + end + + def down + remove_concurrent_index_by_name :releases, name: INDEX_NAME + add_concurrent_index :releases, + %i[project_id tag], + name: OLD_INDEX_NAME + end +end diff --git a/db/post_migrate/20220307203459_rename_user_email_lookup_limit_setting_to_search_settings_cleanup.rb b/db/post_migrate/20220307203459_rename_user_email_lookup_limit_setting_to_search_settings_cleanup.rb new file mode 100644 index 00000000000..2d01374780d --- /dev/null +++ b/db/post_migrate/20220307203459_rename_user_email_lookup_limit_setting_to_search_settings_cleanup.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class RenameUserEmailLookupLimitSettingToSearchSettingsCleanup < Gitlab::Database::Migration[1.0] + class ApplicationSetting < ActiveRecord::Base + self.table_name = :application_settings + end + + def up + ApplicationSetting.update_all 'search_rate_limit=user_email_lookup_limit' + end + + def down + ApplicationSetting.update_all 'user_email_lookup_limit=search_rate_limit' + end +end diff --git a/db/post_migrate/20220308000205_drop_old_index_security_ci_builds_on_name_and_id_parser_features.rb b/db/post_migrate/20220308000205_drop_old_index_security_ci_builds_on_name_and_id_parser_features.rb new file mode 100644 index 00000000000..4b895c291d8 --- /dev/null +++ b/db/post_migrate/20220308000205_drop_old_index_security_ci_builds_on_name_and_id_parser_features.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class DropOldIndexSecurityCiBuildsOnNameAndIdParserFeatures < Gitlab::Database::Migration[1.0] + TABLE = "ci_builds" + COLUMNS = %i[name id] + INDEX_NAME = "index_security_ci_builds_on_name_and_id_parser_features_old" + CONSTRAINTS = "(name::text = ANY (ARRAY['container_scanning'::character varying::text, + 'dast'::character varying::text, + 'dependency_scanning'::character varying::text, + 'license_management'::character varying::text, + 'sast'::character varying::text, + 'secret_detection'::character varying::text, + 'coverage_fuzzing'::character varying::text, + 'license_scanning'::character varying::text]) + ) AND type::text = 'Ci::Build'::text" + + disable_ddl_transaction! + + def up + remove_concurrent_index(TABLE, COLUMNS, name: INDEX_NAME, where: CONSTRAINTS) + end + + def down + add_concurrent_index(TABLE, COLUMNS, name: INDEX_NAME, where: CONSTRAINTS) + end +end diff --git a/db/post_migrate/20220308115219_schedule_reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb b/db/post_migrate/20220308115219_schedule_reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb new file mode 100644 index 00000000000..27e7af9a550 --- /dev/null +++ b/db/post_migrate/20220308115219_schedule_reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class ScheduleResetDuplicateCiRunnersTokenEncryptedValuesOnProjects < Gitlab::Database::Migration[1.0] + MIGRATION = 'ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects' + TOKEN_COLUMN_NAME = :runners_token_encrypted + TEMP_INDEX_NAME = "tmp_index_projects_on_id_and_#{TOKEN_COLUMN_NAME}" + BATCH_SIZE = 10_000 + DELAY_INTERVAL = 2.minutes + + disable_ddl_transaction! + + def up + add_concurrent_index :projects, [:id, TOKEN_COLUMN_NAME], where: "#{TOKEN_COLUMN_NAME} IS NOT NULL", unique: false, name: TEMP_INDEX_NAME + + queue_background_migration_jobs_by_range_at_intervals( + Gitlab::BackgroundMigration::ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects::Project.base_query, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + track_jobs: true + ) + end + + def down + remove_concurrent_index_by_name(:projects, name: TEMP_INDEX_NAME) + end +end diff --git a/db/post_migrate/20220308115502_schedule_reset_duplicate_ci_runners_token_values_on_projects.rb b/db/post_migrate/20220308115502_schedule_reset_duplicate_ci_runners_token_values_on_projects.rb new file mode 100644 index 00000000000..f076b0a740e --- /dev/null +++ b/db/post_migrate/20220308115502_schedule_reset_duplicate_ci_runners_token_values_on_projects.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class ScheduleResetDuplicateCiRunnersTokenValuesOnProjects < Gitlab::Database::Migration[1.0] + MIGRATION = 'ResetDuplicateCiRunnersTokenValuesOnProjects' + TOKEN_COLUMN_NAME = :runners_token + TEMP_INDEX_NAME = "tmp_index_projects_on_id_and_#{TOKEN_COLUMN_NAME}" + BATCH_SIZE = 10_000 + DELAY_INTERVAL = 2.minutes + + disable_ddl_transaction! + + def up + add_concurrent_index :projects, [:id, TOKEN_COLUMN_NAME], where: "#{TOKEN_COLUMN_NAME} IS NOT NULL", unique: false, name: TEMP_INDEX_NAME + + queue_background_migration_jobs_by_range_at_intervals( + Gitlab::BackgroundMigration::ResetDuplicateCiRunnersTokenValuesOnProjects::Project.base_query, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + track_jobs: true + ) + end + + def down + remove_concurrent_index_by_name(:projects, name: TEMP_INDEX_NAME) + end +end diff --git a/db/post_migrate/20220309084838_remove_external_pull_request_tracking.rb b/db/post_migrate/20220309084838_remove_external_pull_request_tracking.rb new file mode 100644 index 00000000000..6159f9d6822 --- /dev/null +++ b/db/post_migrate/20220309084838_remove_external_pull_request_tracking.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class RemoveExternalPullRequestTracking < Gitlab::Database::Migration[1.0] + include Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers + + enable_lock_retries! + + def up + untrack_record_deletions(:external_pull_requests) + end + + def down + track_record_deletions(:external_pull_requests) + end +end diff --git a/db/post_migrate/20220309084954_remove_leftover_external_pull_request_deletions.rb b/db/post_migrate/20220309084954_remove_leftover_external_pull_request_deletions.rb new file mode 100644 index 00000000000..ea9fd6b28c2 --- /dev/null +++ b/db/post_migrate/20220309084954_remove_leftover_external_pull_request_deletions.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class RemoveLeftoverExternalPullRequestDeletions < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + def up + # Delete all pending record deletions in the public.external_pull_requests until + # there are no more rows left. + loop do + result = execute <<~SQL + DELETE FROM "loose_foreign_keys_deleted_records" + WHERE + ("loose_foreign_keys_deleted_records"."partition", "loose_foreign_keys_deleted_records"."id") IN ( + SELECT "loose_foreign_keys_deleted_records"."partition", "loose_foreign_keys_deleted_records"."id" + FROM "loose_foreign_keys_deleted_records" + WHERE + "loose_foreign_keys_deleted_records"."fully_qualified_table_name" = 'public.external_pull_requests' AND + "loose_foreign_keys_deleted_records"."status" = 1 + LIMIT 100 + ) + SQL + + break if result.cmd_tuples == 0 + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220309154855_add_index_on_issues_closed_incidents.rb b/db/post_migrate/20220309154855_add_index_on_issues_closed_incidents.rb new file mode 100644 index 00000000000..e9a2c1c85f2 --- /dev/null +++ b/db/post_migrate/20220309154855_add_index_on_issues_closed_incidents.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddIndexOnIssuesClosedIncidents < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + INDEX_NAME = 'index_on_issues_closed_incidents_by_project_id_and_closed_at' + + def up + add_concurrent_index :issues, [:project_id, :closed_at], where: "issue_type = 1 AND state_id = 2", name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :issues, INDEX_NAME + end +end diff --git a/db/post_migrate/20220310095341_add_async_index_ci_job_artifacts_project_id_created_at.rb b/db/post_migrate/20220310095341_add_async_index_ci_job_artifacts_project_id_created_at.rb new file mode 100644 index 00000000000..919e834a783 --- /dev/null +++ b/db/post_migrate/20220310095341_add_async_index_ci_job_artifacts_project_id_created_at.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddAsyncIndexCiJobArtifactsProjectIdCreatedAt < Gitlab::Database::Migration[1.0] + INDEX_NAME = 'index_ci_job_artifacts_on_id_project_id_and_created_at' + + def up + prepare_async_index :ci_job_artifacts, [:project_id, :created_at, :id], name: INDEX_NAME + end + + def down + unprepare_async_index_by_name :ci_job_artifacts, INDEX_NAME + end +end diff --git a/db/post_migrate/20220310134207_add_index_project_id_and_released_at_and_id_on_releases.rb b/db/post_migrate/20220310134207_add_index_project_id_and_released_at_and_id_on_releases.rb new file mode 100644 index 00000000000..da928f3ec8f --- /dev/null +++ b/db/post_migrate/20220310134207_add_index_project_id_and_released_at_and_id_on_releases.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddIndexProjectIdAndReleasedAtAndIdOnReleases < Gitlab::Database::Migration[1.0] + INDEX_NAME = 'index_releases_on_project_id_and_released_at_and_id' + + disable_ddl_transaction! + + def up + add_concurrent_index :releases, [:project_id, :released_at, :id], + name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :releases, INDEX_NAME + end +end diff --git a/db/post_migrate/20220310141349_remove_dependency_list_usage_data_from_redis.rb b/db/post_migrate/20220310141349_remove_dependency_list_usage_data_from_redis.rb new file mode 100644 index 00000000000..3c1e6714529 --- /dev/null +++ b/db/post_migrate/20220310141349_remove_dependency_list_usage_data_from_redis.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class RemoveDependencyListUsageDataFromRedis < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + def up + Gitlab::Redis::SharedState.with { |r| r.del("DEPENDENCY_LIST_USAGE_COUNTER") } + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220314162342_add_index_ci_job_artifacts_project_id_created_at.rb b/db/post_migrate/20220314162342_add_index_ci_job_artifacts_project_id_created_at.rb new file mode 100644 index 00000000000..7241fd54cb3 --- /dev/null +++ b/db/post_migrate/20220314162342_add_index_ci_job_artifacts_project_id_created_at.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddIndexCiJobArtifactsProjectIdCreatedAt < Gitlab::Database::Migration[1.0] + INDEX_NAME = 'index_ci_job_artifacts_on_id_project_id_and_created_at' + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_job_artifacts, [:project_id, :created_at, :id], name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME + end +end -- cgit v1.2.1