diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-05-20 14:34:42 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-05-20 14:34:42 +0000 |
commit | 9f46488805e86b1bc341ea1620b866016c2ce5ed (patch) | |
tree | f9748c7e287041e37d6da49e0a29c9511dc34768 /db/post_migrate | |
parent | dfc92d081ea0332d69c8aca2f0e745cb48ae5e6d (diff) | |
download | gitlab-ce-9f46488805e86b1bc341ea1620b866016c2ce5ed.tar.gz |
Add latest changes from gitlab-org/gitlab@13-0-stable-ee
Diffstat (limited to 'db/post_migrate')
38 files changed, 827 insertions, 11 deletions
diff --git a/db/post_migrate/20181008200441_remove_circuit_breaker.rb b/db/post_migrate/20181008200441_remove_circuit_breaker.rb index 378692e8886..09491038e50 100644 --- a/db/post_migrate/20181008200441_remove_circuit_breaker.rb +++ b/db/post_migrate/20181008200441_remove_circuit_breaker.rb @@ -24,7 +24,9 @@ class RemoveCircuitBreaker < ActiveRecord::Migration[4.2] def down CIRCUIT_BREAKER_COLUMS_WITH_DEFAULT.each do |column, default| + # rubocop:disable Migration/AddColumnWithDefault add_column_with_default(:application_settings, column, :integer, default: default) unless column_exists?(:application_settings, column) + # rubocop:enable Migration/AddColumnWithDefault end end end diff --git a/db/post_migrate/20181013005024_remove_koding_from_application_settings.rb b/db/post_migrate/20181013005024_remove_koding_from_application_settings.rb index b6e5473e896..550ad94f4ab 100644 --- a/db/post_migrate/20181013005024_remove_koding_from_application_settings.rb +++ b/db/post_migrate/20181013005024_remove_koding_from_application_settings.rb @@ -12,6 +12,6 @@ class RemoveKodingFromApplicationSettings < ActiveRecord::Migration[4.2] def down add_column :application_settings, :koding_enabled, :boolean # rubocop:disable Migration/SaferBooleanColumn - add_column :application_settings, :koding_url, :string # rubocop:disable Migration/AddLimitToStringColumns + add_column :application_settings, :koding_url, :string end end diff --git a/db/post_migrate/20190404231137_remove_alternate_url_from_geo_nodes.rb b/db/post_migrate/20190404231137_remove_alternate_url_from_geo_nodes.rb index 8e7ef0ec54f..785ceb2fb28 100644 --- a/db/post_migrate/20190404231137_remove_alternate_url_from_geo_nodes.rb +++ b/db/post_migrate/20190404231137_remove_alternate_url_from_geo_nodes.rb @@ -16,6 +16,6 @@ class RemoveAlternateUrlFromGeoNodes < ActiveRecord::Migration[5.0] end def down - add_column :geo_nodes, :alternate_url, :string # rubocop:disable Migration/AddLimitToStringColumns + add_column :geo_nodes, :alternate_url, :string end end diff --git a/db/post_migrate/20190625184066_remove_sentry_from_application_settings.rb b/db/post_migrate/20190625184066_remove_sentry_from_application_settings.rb index 9d71bfafffb..7a0923aabd8 100644 --- a/db/post_migrate/20190625184066_remove_sentry_from_application_settings.rb +++ b/db/post_migrate/20190625184066_remove_sentry_from_application_settings.rb @@ -28,11 +28,13 @@ class RemoveSentryFromApplicationSettings < ActiveRecord::Migration[5.0] def down SENTRY_ENABLED_COLUMNS.each do |column| + # rubocop:disable Migration/AddColumnWithDefault add_column_with_default(:application_settings, column, :boolean, default: false, allow_null: false) unless column_exists?(:application_settings, column) + # rubocop:enable Migration/AddColumnWithDefault end SENTRY_DSN_COLUMNS.each do |column| - add_column(:application_settings, column, :string) unless column_exists?(:application_settings, column) # rubocop:disable Migration/AddLimitToStringColumns + add_column(:application_settings, column, :string) unless column_exists?(:application_settings, column) end end end diff --git a/db/post_migrate/20191030193050_remove_pendo_from_application_settings.rb b/db/post_migrate/20191030193050_remove_pendo_from_application_settings.rb index 33bbe6f8ea7..c1a1cc01aa4 100644 --- a/db/post_migrate/20191030193050_remove_pendo_from_application_settings.rb +++ b/db/post_migrate/20191030193050_remove_pendo_from_application_settings.rb @@ -13,7 +13,7 @@ class RemovePendoFromApplicationSettings < ActiveRecord::Migration[5.2] end def down - add_column_with_default :application_settings, :pendo_enabled, :boolean, default: false, allow_null: false + add_column_with_default :application_settings, :pendo_enabled, :boolean, default: false, allow_null: false # rubocop:disable Migration/AddColumnWithDefault add_column :application_settings, :pendo_url, :string, limit: 255 end end diff --git a/db/post_migrate/20191128162854_drop_project_ci_cd_settings_merge_trains_enabled.rb b/db/post_migrate/20191128162854_drop_project_ci_cd_settings_merge_trains_enabled.rb index c2e6792e611..7f91d5112b3 100644 --- a/db/post_migrate/20191128162854_drop_project_ci_cd_settings_merge_trains_enabled.rb +++ b/db/post_migrate/20191128162854_drop_project_ci_cd_settings_merge_trains_enabled.rb @@ -12,6 +12,8 @@ class DropProjectCiCdSettingsMergeTrainsEnabled < ActiveRecord::Migration[5.2] end def down + # rubocop:disable Migration/AddColumnWithDefault add_column_with_default :project_ci_cd_settings, :merge_trains_enabled, :boolean, default: false, allow_null: true # rubocop:disable Migration/UpdateLargeTable + # rubocop:enable Migration/AddColumnWithDefault end end diff --git a/db/post_migrate/20191202031812_drop_operations_feature_flags_clients_token.rb b/db/post_migrate/20191202031812_drop_operations_feature_flags_clients_token.rb index bda461af7bc..93cef322f02 100644 --- a/db/post_migrate/20191202031812_drop_operations_feature_flags_clients_token.rb +++ b/db/post_migrate/20191202031812_drop_operations_feature_flags_clients_token.rb @@ -14,7 +14,7 @@ class DropOperationsFeatureFlagsClientsToken < ActiveRecord::Migration[5.2] def down unless column_exists?(:operations_feature_flags_clients, :token) - add_column :operations_feature_flags_clients, :token, :string # rubocop:disable Migration/AddLimitToStringColumns + add_column :operations_feature_flags_clients, :token, :string end add_concurrent_index :operations_feature_flags_clients, [:project_id, :token], unique: true, diff --git a/db/post_migrate/20200120083607_remove_storage_version_column_from_snippets.rb b/db/post_migrate/20200120083607_remove_storage_version_column_from_snippets.rb index e94dc75e65c..62bb3f46cae 100644 --- a/db/post_migrate/20200120083607_remove_storage_version_column_from_snippets.rb +++ b/db/post_migrate/20200120083607_remove_storage_version_column_from_snippets.rb @@ -19,7 +19,7 @@ class RemoveStorageVersionColumnFromSnippets < ActiveRecord::Migration[5.2] def down return if column_exists?(:snippets, :storage_version) - add_column_with_default( + add_column_with_default( # rubocop:disable Migration/AddColumnWithDefault :snippets, :storage_version, :integer, diff --git a/db/post_migrate/20200128210353_cleanup_optimistic_locking_nulls.rb b/db/post_migrate/20200128210353_cleanup_optimistic_locking_nulls.rb index 8bc037c7333..5c172159561 100644 --- a/db/post_migrate/20200128210353_cleanup_optimistic_locking_nulls.rb +++ b/db/post_migrate/20200128210353_cleanup_optimistic_locking_nulls.rb @@ -31,7 +31,7 @@ class CleanupOptimisticLockingNulls < ActiveRecord::Migration[5.2] 'CleanupOptimisticLockingNulls', 2.minutes, batch_size: BATCH_SIZE, - other_arguments: [table] + other_job_arguments: [table] ) end end diff --git a/db/post_migrate/20200212052620_readd_template_column_to_services.rb b/db/post_migrate/20200212052620_readd_template_column_to_services.rb index 2b0d26b2ad4..c636cb0a07b 100644 --- a/db/post_migrate/20200212052620_readd_template_column_to_services.rb +++ b/db/post_migrate/20200212052620_readd_template_column_to_services.rb @@ -15,7 +15,7 @@ class ReaddTemplateColumnToServices < ActiveRecord::Migration[6.0] # to production, so we should be okay to re-add it without worrying # about doing a data migration. If we needed to restore the value # of `template`, we would look for entries with `project_id IS NULL`. - add_column_with_default :services, :template, :boolean, default: false, allow_null: true + add_column_with_default :services, :template, :boolean, default: false, allow_null: true # rubocop:disable Migration/AddColumnWithDefault end # rubocop:enable Migration/UpdateLargeTable diff --git a/db/post_migrate/20200217210353_cleanup_optimistic_locking_nulls_pt2.rb b/db/post_migrate/20200217210353_cleanup_optimistic_locking_nulls_pt2.rb new file mode 100644 index 00000000000..1ec3e817e8e --- /dev/null +++ b/db/post_migrate/20200217210353_cleanup_optimistic_locking_nulls_pt2.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class CleanupOptimisticLockingNullsPt2 < ActiveRecord::Migration[5.2] + def change + # no-op: the MR that contained this migration was reverted + end +end diff --git a/db/post_migrate/20200219193058_remove_state_from_issues.rb b/db/post_migrate/20200219193058_remove_state_from_issues.rb index ac27a9a9b69..007ba600ce7 100644 --- a/db/post_migrate/20200219193058_remove_state_from_issues.rb +++ b/db/post_migrate/20200219193058_remove_state_from_issues.rb @@ -18,7 +18,7 @@ class RemoveStateFromIssues < ActiveRecord::Migration[6.0] return if issue_state_column_exists? with_lock_retries do - add_column :issues, :state, :string # rubocop:disable Migration/AddLimitToStringColumns + add_column :issues, :state, :string end end diff --git a/db/post_migrate/20200219193117_remove_state_from_merge_requests.rb b/db/post_migrate/20200219193117_remove_state_from_merge_requests.rb index c99a732f37b..384a694b549 100644 --- a/db/post_migrate/20200219193117_remove_state_from_merge_requests.rb +++ b/db/post_migrate/20200219193117_remove_state_from_merge_requests.rb @@ -18,7 +18,7 @@ class RemoveStateFromMergeRequests < ActiveRecord::Migration[6.0] return if merge_requests_state_column_exists? with_lock_retries do - add_column :merge_requests, :state, :string # rubocop:disable Migration/AddLimitToStringColumns + add_column :merge_requests, :state, :string end end diff --git a/db/post_migrate/20200221142216_remove_repository_storage_from_snippets.rb b/db/post_migrate/20200221142216_remove_repository_storage_from_snippets.rb index fb8721a45b8..f9ef985218b 100644 --- a/db/post_migrate/20200221142216_remove_repository_storage_from_snippets.rb +++ b/db/post_migrate/20200221142216_remove_repository_storage_from_snippets.rb @@ -16,7 +16,7 @@ class RemoveRepositoryStorageFromSnippets < ActiveRecord::Migration[6.0] def down return if column_exists?(:snippets, :repository_storage) - add_column_with_default( + add_column_with_default( # rubocop:disable Migration/AddColumnWithDefault :snippets, :repository_storage, :string, diff --git a/db/post_migrate/20200310215714_migrate_saml_identities_to_scim_identities.rb b/db/post_migrate/20200310215714_migrate_saml_identities_to_scim_identities.rb new file mode 100644 index 00000000000..e2ec7b62d31 --- /dev/null +++ b/db/post_migrate/20200310215714_migrate_saml_identities_to_scim_identities.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class MigrateSamlIdentitiesToScimIdentities < ActiveRecord::Migration[6.0] + DOWNTIME = false + + class Identity < ActiveRecord::Base + self.table_name = 'identities' + + include ::EachBatch + end + + def up + Identity + .joins('INNER JOIN saml_providers ON saml_providers.id = identities.saml_provider_id') + .where('saml_providers.group_id IN (SELECT group_id FROM scim_oauth_access_tokens)') + .select('identities.extern_uid, identities.user_id, saml_providers.group_id, TRUE AS active, + identities.created_at, CURRENT_TIMESTAMP AS updated_at') + .each_batch do |batch| + data_to_insert = batch.map do |record| + record.attributes.extract!("extern_uid", "user_id", "group_id", "active", "created_at", "updated_at") + end + + Gitlab::Database.bulk_insert(:scim_identities, data_to_insert, on_conflict: :do_nothing) + end + end + + def down + end +end diff --git a/db/post_migrate/20200312134637_backfill_environment_id_on_deployment_merge_requests.rb b/db/post_migrate/20200312134637_backfill_environment_id_on_deployment_merge_requests.rb new file mode 100644 index 00000000000..77cb1ae8508 --- /dev/null +++ b/db/post_migrate/20200312134637_backfill_environment_id_on_deployment_merge_requests.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class BackfillEnvironmentIdOnDeploymentMergeRequests < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + # no-op + + # this migration is deleted because there is no foreign key for + # deployments.environment_id and this caused a failure upgrading + # deployments_merge_requests.environment_id + # + # Details on the following issues: + # * https://gitlab.com/gitlab-org/gitlab/-/issues/217191 + # * https://gitlab.com/gitlab-org/gitlab/-/issues/26229 + end + + def down + # no-op + + # this migration is designed to delete duplicated data + end +end diff --git a/db/post_migrate/20200401091051_remove_reference_columns_from_resource_milestone_events.rb b/db/post_migrate/20200401091051_remove_reference_columns_from_resource_milestone_events.rb new file mode 100644 index 00000000000..639ab93cf18 --- /dev/null +++ b/db/post_migrate/20200401091051_remove_reference_columns_from_resource_milestone_events.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class RemoveReferenceColumnsFromResourceMilestoneEvents < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def change + remove_column :resource_milestone_events, :reference, :text + remove_column :resource_milestone_events, :reference_html, :text + remove_column :resource_milestone_events, :cached_markdown_version, :integer + end +end diff --git a/db/post_migrate/20200403132349_remove_old_index_pages_domains_need_auto_ssl_renewal.rb b/db/post_migrate/20200403132349_remove_old_index_pages_domains_need_auto_ssl_renewal.rb new file mode 100644 index 00000000000..b36dce188df --- /dev/null +++ b/db/post_migrate/20200403132349_remove_old_index_pages_domains_need_auto_ssl_renewal.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveOldIndexPagesDomainsNeedAutoSslRenewal < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + INDEX_NAME = 'index_pages_domains_need_auto_ssl_renewal' + + disable_ddl_transaction! + + def up + remove_concurrent_index(:pages_domains, [:certificate_source, :certificate_valid_not_after], + where: "auto_ssl_enabled = true", name: INDEX_NAME) + end + + def down + add_concurrent_index(:pages_domains, [:certificate_source, :certificate_valid_not_after], + where: "auto_ssl_enabled = true", name: INDEX_NAME) + end +end diff --git a/db/post_migrate/20200420094444_backfill_snippet_repositories.rb b/db/post_migrate/20200420094444_backfill_snippet_repositories.rb new file mode 100644 index 00000000000..452a1a5330f --- /dev/null +++ b/db/post_migrate/20200420094444_backfill_snippet_repositories.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class BackfillSnippetRepositories < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INTERVAL = 3.minutes + BATCH_SIZE = 100 + MIGRATION = 'BackfillSnippetRepositories' + + disable_ddl_transaction! + + class Snippet < ActiveRecord::Base + include EachBatch + + self.table_name = 'snippets' + self.inheritance_column = :_type_disabled + end + + def up + queue_background_migration_jobs_by_range_at_intervals(Snippet, + MIGRATION, + INTERVAL, + batch_size: BATCH_SIZE) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200420162730_remove_additional_application_settings_rows.rb b/db/post_migrate/20200420162730_remove_additional_application_settings_rows.rb new file mode 100644 index 00000000000..e4a0ec1eb4a --- /dev/null +++ b/db/post_migrate/20200420162730_remove_additional_application_settings_rows.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class RemoveAdditionalApplicationSettingsRows < ActiveRecord::Migration[6.0] + class ApplicationSetting < ActiveRecord::Base + self.table_name = 'application_settings' + end + + def up + return if ApplicationSetting.count == 1 + + execute "DELETE from application_settings WHERE id NOT IN (SELECT MAX(id) FROM application_settings);" + end + + def down + # no changes + end +end diff --git a/db/post_migrate/20200424043515_drop_namespaces_plan_id.rb b/db/post_migrate/20200424043515_drop_namespaces_plan_id.rb new file mode 100644 index 00000000000..16a56b16e5a --- /dev/null +++ b/db/post_migrate/20200424043515_drop_namespaces_plan_id.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class DropNamespacesPlanId < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + with_lock_retries do + remove_column :namespaces, :plan_id + end + end + + def down + unless column_exists?(:namespaces, :plan_id) + with_lock_retries do + add_column :namespaces, :plan_id, :integer + end + end + + add_concurrent_index :namespaces, :plan_id + add_concurrent_foreign_key :namespaces, :plans, column: :plan_id, on_delete: :nullify + end +end diff --git a/db/post_migrate/20200427064130_cleanup_optimistic_locking_nulls_pt2_fixed.rb b/db/post_migrate/20200427064130_cleanup_optimistic_locking_nulls_pt2_fixed.rb new file mode 100644 index 00000000000..63f85fc7156 --- /dev/null +++ b/db/post_migrate/20200427064130_cleanup_optimistic_locking_nulls_pt2_fixed.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +class CleanupOptimisticLockingNullsPt2Fixed < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + TABLES = %w(ci_stages ci_builds ci_pipelines).freeze + BATCH_SIZE = 10_000 + + def declare_class(table) + Class.new(ActiveRecord::Base) do + include EachBatch + + self.table_name = table + self.inheritance_column = :_type_disabled # Disable STI + end + end + + def up + last_table_final_delay = 0 + + TABLES.each do |table| + # cleanup wrong index created in the previous migration, it might be there on staging + remove_concurrent_index table.to_sym, :lock_version, where: "lock_version IS NULL" + + add_concurrent_index table.to_sym, :id, where: "lock_version IS NULL", name: "tmp_index_#{table}_lock_version" + + last_table_final_delay = queue_background_migration_jobs_by_range_at_intervals( + declare_class(table).where(lock_version: nil), + 'CleanupOptimisticLockingNulls', + 2.minutes, + batch_size: BATCH_SIZE, + other_job_arguments: [table], + initial_delay: last_table_final_delay + ) + end + end + + def down + TABLES.each do |table| + remove_concurrent_index table.to_sym, :id, where: "lock_version IS NULL", name: "tmp_index_#{table}_lock_version" + end + end +end diff --git a/db/post_migrate/20200428134356_remove_elastic_experimental_indexer_from_application_settings.rb b/db/post_migrate/20200428134356_remove_elastic_experimental_indexer_from_application_settings.rb new file mode 100644 index 00000000000..a9baf6fd8e3 --- /dev/null +++ b/db/post_migrate/20200428134356_remove_elastic_experimental_indexer_from_application_settings.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true +class RemoveElasticExperimentalIndexerFromApplicationSettings < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def change + remove_column :application_settings, :elasticsearch_experimental_indexer, :boolean + end +end diff --git a/db/post_migrate/20200429002150_cleanup_sprints_state_rename.rb b/db/post_migrate/20200429002150_cleanup_sprints_state_rename.rb new file mode 100644 index 00000000000..7f67a55a19d --- /dev/null +++ b/db/post_migrate/20200429002150_cleanup_sprints_state_rename.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class CleanupSprintsStateRename < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + cleanup_concurrent_column_rename :sprints, :state, :state_enum + end + + def down + undo_cleanup_concurrent_column_rename :sprints, :state, :state_enum + end +end diff --git a/db/post_migrate/20200506085748_update_undefined_confidence_from_occurrences.rb b/db/post_migrate/20200506085748_update_undefined_confidence_from_occurrences.rb new file mode 100644 index 00000000000..06c82ad404b --- /dev/null +++ b/db/post_migrate/20200506085748_update_undefined_confidence_from_occurrences.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class UpdateUndefinedConfidenceFromOccurrences < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + INDEX_NAME = 'index_vulnerability_occurrences_on_id_and_confidence_eq_zero' + DOWNTIME = false + + disable_ddl_transaction! + BATCH_SIZE = 1_000 + INTERVAL = 2.minutes + + # 286_159 records to be updated on GitLab.com + def up + # create temporary index for undefined vulnerabilities + add_concurrent_index(:vulnerability_occurrences, :id, where: 'confidence = 0', name: INDEX_NAME) + + return unless Gitlab.ee? + + migration = Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel + migration_name = migration.to_s.demodulize + relation = migration::Occurrence.undefined_confidence + queue_background_migration_jobs_by_range_at_intervals(relation, + migration_name, + INTERVAL, + batch_size: BATCH_SIZE) + end + + def down + # no-op + # temporary index is to be dropped in a different migration in an upcoming release + remove_concurrent_index(:vulnerability_occurrences, :id, where: 'confidence = 0', name: INDEX_NAME) + # This migration can not be reversed because we can not know which records had undefined confidence + end +end diff --git a/db/post_migrate/20200506125731_cleanup_user_highest_roles_population.rb b/db/post_migrate/20200506125731_cleanup_user_highest_roles_population.rb new file mode 100644 index 00000000000..5e613228c56 --- /dev/null +++ b/db/post_migrate/20200506125731_cleanup_user_highest_roles_population.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class CleanupUserHighestRolesPopulation < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_for_migrating_user_highest_roles_table' + + disable_ddl_transaction! + + def up + Gitlab::BackgroundMigration.steal('PopulateUserHighestRolesTable') + + remove_concurrent_index(:users, :id, name: INDEX_NAME) + end + + def down + add_concurrent_index(:users, + :id, + where: "state = 'active' AND user_type IS NULL AND bot_type IS NULL AND ghost IS NOT TRUE", + name: INDEX_NAME) + end +end diff --git a/db/post_migrate/20200506154421_migrate_scim_identities_to_saml_for_new_users.rb b/db/post_migrate/20200506154421_migrate_scim_identities_to_saml_for_new_users.rb new file mode 100644 index 00000000000..718e788aad7 --- /dev/null +++ b/db/post_migrate/20200506154421_migrate_scim_identities_to_saml_for_new_users.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class MigrateScimIdentitiesToSamlForNewUsers < ActiveRecord::Migration[6.0] + DOWNTIME = false + + class ScimIdentity < ActiveRecord::Base + self.table_name = 'scim_identities' + + belongs_to :user + + include ::EachBatch + end + + class Identity < ActiveRecord::Base + self.table_name = 'identities' + + belongs_to :saml_provider + end + + def up + users_with_saml_provider = Identity.select('user_id').joins(:saml_provider) + + ScimIdentity.each_batch do |relation| + identity_records = relation + .select("scim_identities.extern_uid, 'group_saml', scim_identities.user_id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, saml_providers.id") + .joins(:user) + .joins('inner join saml_providers on saml_providers.group_id=scim_identities.group_id') + .where("date_trunc('second',scim_identities.created_at) at time zone 'UTC' = date_trunc('second',users.created_at)") + .where.not(user_id: users_with_saml_provider) + + execute "insert into identities (extern_uid, provider, user_id, created_at, updated_at, saml_provider_id) #{identity_records.to_sql} on conflict do nothing" + end + end + + def down + end +end diff --git a/db/post_migrate/20200508091106_remove_bot_type.rb b/db/post_migrate/20200508091106_remove_bot_type.rb new file mode 100644 index 00000000000..2afcf5308e7 --- /dev/null +++ b/db/post_migrate/20200508091106_remove_bot_type.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class RemoveBotType < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name :users, 'index_users_on_bot_type' + + with_lock_retries do + remove_column :users, :bot_type + end + end + + def down + unless column_exists?(:users, :bot_type) + with_lock_retries do + add_column :users, :bot_type, :integer, limit: 2 # rubocop:disable Migration/AddColumnsToWideTables + end + end + + execute 'UPDATE users set bot_type = user_type WHERE user_type IN(1,2,3,6)' + + add_concurrent_index :users, :bot_type + end +end diff --git a/db/post_migrate/20200511080113_add_projects_foreign_key_to_namespaces.rb b/db/post_migrate/20200511080113_add_projects_foreign_key_to_namespaces.rb new file mode 100644 index 00000000000..a7f67a3b5cd --- /dev/null +++ b/db/post_migrate/20200511080113_add_projects_foreign_key_to_namespaces.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class AddProjectsForeignKeyToNamespaces < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + FK_NAME = 'fk_projects_namespace_id' + + def up + with_lock_retries do + add_foreign_key( + :projects, + :namespaces, + column: :namespace_id, + on_delete: :restrict, + validate: false, + name: FK_NAME + ) + end + end + + def down + with_lock_retries do + remove_foreign_key_if_exists :projects, column: :namespace_id, name: FK_NAME + end + end +end diff --git a/db/post_migrate/20200511083541_cleanup_projects_with_missing_namespace.rb b/db/post_migrate/20200511083541_cleanup_projects_with_missing_namespace.rb new file mode 100644 index 00000000000..442acfc6d16 --- /dev/null +++ b/db/post_migrate/20200511083541_cleanup_projects_with_missing_namespace.rb @@ -0,0 +1,263 @@ +# frozen_string_literal: true + +# rubocop:disable Migration/PreventStrings + +# This migration cleans up Projects that were orphaned when their namespace was deleted +# Instead of deleting them, we: +# - Find (or create) the Ghost User +# - Create (if not already exists) a `lost-and-found` group owned by the Ghost User +# - Find orphaned projects --> namespace_id can not be found in namespaces +# - Move the orphaned projects to the `lost-and-found` group +# (while making them private and setting `archived=true`) +# +# On GitLab.com (2020-05-11) this migration will update 66 orphaned projects +class CleanupProjectsWithMissingNamespace < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + VISIBILITY_PRIVATE = 0 + ACCESS_LEVEL_OWNER = 50 + + # The batch size of projects to check in each iteration + # We expect the selectivity for orphaned projects to be very low: + # (66 orphaned projects out of a total 13.6M) + # so 10K should be a safe choice + BATCH_SIZE = 10000 + + disable_ddl_transaction! + + class UserDetail < ActiveRecord::Base + self.table_name = 'user_details' + + belongs_to :user, class_name: 'CleanupProjectsWithMissingNamespace::User' + end + + class User < ActiveRecord::Base + self.table_name = 'users' + + LOST_AND_FOUND_GROUP = 'lost-and-found' + USER_TYPE_GHOST = 5 + DEFAULT_PROJECTS_LIMIT = 100000 + + default_value_for :admin, false + default_value_for :can_create_group, true # we need this to create the group + default_value_for :can_create_team, false + default_value_for :project_view, :files + default_value_for :notified_of_own_activity, false + default_value_for :preferred_language, I18n.default_locale + + has_one :user_detail, class_name: 'CleanupProjectsWithMissingNamespace::UserDetail' + has_one :namespace, -> { where(type: nil) }, + foreign_key: :owner_id, inverse_of: :owner, autosave: true, + class_name: 'CleanupProjectsWithMissingNamespace::Namespace' + + before_save :ensure_namespace_correct + before_save :ensure_bio_is_assigned_to_user_details, if: :bio_changed? + + enum project_view: { readme: 0, activity: 1, files: 2 } + + def ensure_namespace_correct + if namespace + namespace.path = username if username_changed? + namespace.name = name if name_changed? + else + build_namespace(path: username, name: name) + end + end + + def ensure_bio_is_assigned_to_user_details + return if Feature.disabled?(:migrate_bio_to_user_details, default_enabled: true) + + user_detail.bio = bio.to_s[0...255] + end + + def user_detail + super.presence || build_user_detail + end + + # Return (or create if necessary) the `lost-and-found` group + def lost_and_found_group + existing_lost_and_found_group || Group.create_unique_group(self, LOST_AND_FOUND_GROUP) + end + + def existing_lost_and_found_group + # There should only be one Group for User Ghost starting with LOST_AND_FOUND_GROUP + Group + .joins('INNER JOIN members ON namespaces.id = members.source_id') + .where('namespaces.type = ?', 'Group') + .where('members.type = ?', 'GroupMember') + .where('members.source_type = ?', 'Namespace') + .where('members.user_id = ?', self.id) + .where('members.requested_at IS NULL') + .where('members.access_level = ?', ACCESS_LEVEL_OWNER) + .find_by(Group.arel_table[:name].matches("#{LOST_AND_FOUND_GROUP}%")) + end + + class << self + # Return (or create if necessary) the ghost user + def ghost + email = 'ghost%s@example.com' + + unique_internal(where(user_type: USER_TYPE_GHOST), 'ghost', email) do |u| + u.bio = _('This is a "Ghost User", created to hold all issues authored by users that have since been deleted. This user cannot be removed.') + u.name = 'Ghost User' + end + end + + def unique_internal(scope, username, email_pattern, &block) + scope.first || create_unique_internal(scope, username, email_pattern, &block) + end + + def create_unique_internal(scope, username, email_pattern, &creation_block) + # Since we only want a single one of these in an instance, we use an + # exclusive lease to ensure that this block is never run concurrently. + lease_key = "user:unique_internal:#{username}" + lease = Gitlab::ExclusiveLease.new(lease_key, timeout: 1.minute.to_i) + + until uuid = lease.try_obtain + # Keep trying until we obtain the lease. To prevent hammering Redis too + # much we'll wait for a bit between retries. + sleep(1) + end + + # Recheck if the user is already present. One might have been + # added between the time we last checked (first line of this method) + # and the time we acquired the lock. + existing_user = uncached { scope.first } + return existing_user if existing_user.present? + + uniquify = Uniquify.new + + username = uniquify.string(username) { |s| User.find_by_username(s) } + + email = uniquify.string(-> (n) { Kernel.sprintf(email_pattern, n) }) do |s| + User.find_by_email(s) + end + + User.create!( + username: username, + email: email, + user_type: USER_TYPE_GHOST, + projects_limit: DEFAULT_PROJECTS_LIMIT, + state: :active, + &creation_block + ) + ensure + Gitlab::ExclusiveLease.cancel(lease_key, uuid) + end + end + end + + class Namespace < ActiveRecord::Base + self.table_name = 'namespaces' + + belongs_to :owner, class_name: 'CleanupProjectsWithMissingNamespace::User' + end + + class Group < Namespace + # Disable STI to allow us to manually set "type = 'Group'" + # Otherwise rails forces "type = CleanupProjectsWithMissingNamespace::Group" + self.inheritance_column = :_type_disabled + + def self.create_unique_group(user, group_name) + # 'lost-and-found' may be already defined, find a unique one + group_name = Uniquify.new.string(group_name) do |str| + Group.where(parent_id: nil, name: str).exists? + end + + group = Group.create!( + name: group_name, + path: group_name, + type: 'Group', + description: 'Group to store orphaned projects', + visibility_level: VISIBILITY_PRIVATE + ) + + # No need to create a route for the lost-and-found group + + GroupMember.add_user(group, user, ACCESS_LEVEL_OWNER) + + group + end + end + + class Member < ActiveRecord::Base + self.table_name = 'members' + end + + class GroupMember < Member + NOTIFICATION_SETTING_GLOBAL = 3 + + # Disable STI to allow us to manually set "type = 'GroupMember'" + # Otherwise rails forces "type = CleanupProjectsWithMissingNamespace::GroupMember" + self.inheritance_column = :_type_disabled + + def self.add_user(source, user, access_level) + GroupMember.create!( + type: 'GroupMember', + source_id: source.id, + user_id: user.id, + source_type: 'Namespace', + access_level: access_level, + notification_level: NOTIFICATION_SETTING_GLOBAL + ) + end + end + + class Project < ActiveRecord::Base + self.table_name = 'projects' + + include ::EachBatch + + def self.without_namespace + where( + 'NOT EXISTS ( + SELECT 1 + FROM namespaces + WHERE projects.namespace_id = namespaces.id + )' + ) + end + end + + def up + # Reset the column information of all the models that update the database + # to ensure the Active Record's knowledge of the table structure is current + User.reset_column_information + Namespace.reset_column_information + Member.reset_column_information + Project.reset_column_information + + # Find or Create the ghost user + ghost_user = User.ghost + + # Find or Create the `lost-and-found` + lost_and_found = ghost_user.lost_and_found_group + + # With BATCH_SIZE=10K and projects.count=13.6M + # ~1360 iterations will be run: + # - each requires on average ~160ms for relation.without_namespace + # - worst case scenario is that 66 of those batches will trigger an update (~200ms each) + # In general, we expect less than 5% (=66/13.6M x 10K) to trigger an update + # Expected total run time: ~235 seconds (== 220 seconds + 14 seconds) + Project.each_batch(of: BATCH_SIZE) do |relation| + relation.without_namespace.update_all <<~SQL + namespace_id = #{lost_and_found.id}, + archived = TRUE, + visibility_level = #{VISIBILITY_PRIVATE}, + + -- Names are expected to be unique inside their namespace + -- (uniqueness validation on namespace_id, name) + -- Attach the id to the name and path to make sure that they are unique + name = name || '_' || id, + path = path || '_' || id + SQL + end + end + + def down + # no-op: the original state for those projects was inconsistent + # Also, the original namespace_id for each project is lost during the update + end +end +# rubocop:enable Migration/PreventStrings diff --git a/db/post_migrate/20200511092714_update_undefined_confidence_from_vulnerabilities.rb b/db/post_migrate/20200511092714_update_undefined_confidence_from_vulnerabilities.rb new file mode 100644 index 00000000000..d6611ddbd66 --- /dev/null +++ b/db/post_migrate/20200511092714_update_undefined_confidence_from_vulnerabilities.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class UpdateUndefinedConfidenceFromVulnerabilities < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + INDEX_NAME = 'index_vulnerability_on_id_and_confidence_eq_zero' + DOWNTIME = false + + disable_ddl_transaction! + BATCH_SIZE = 1_000 + INTERVAL = 2.minutes + + # 87_602 records to be updated on GitLab.com + def up + # create temporary index for undefined vulnerabilities + add_concurrent_index(:vulnerabilities, :id, where: 'confidence = 0', name: INDEX_NAME) + + return unless Gitlab.ee? + + migration = Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel + migration_name = migration.to_s.demodulize + relation = migration::Vulnerability.undefined_confidence + queue_background_migration_jobs_by_range_at_intervals(relation, + migration_name, + INTERVAL, + batch_size: BATCH_SIZE) + end + + def down + # no-op + # temporary index is to be dropped in a different migration in an upcoming release + remove_concurrent_index(:vulnerabilities, :id, where: 'confidence = 0', name: INDEX_NAME) + # This migration can not be reversed because we can not know which records had undefined confidence + end +end diff --git a/db/post_migrate/20200511130130_ensure_deprecated_jenkins_service_records_removal.rb b/db/post_migrate/20200511130130_ensure_deprecated_jenkins_service_records_removal.rb new file mode 100644 index 00000000000..4c1f29f8e47 --- /dev/null +++ b/db/post_migrate/20200511130130_ensure_deprecated_jenkins_service_records_removal.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class EnsureDeprecatedJenkinsServiceRecordsRemoval < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + execute <<~SQL.strip + DELETE FROM services WHERE type = 'JenkinsDeprecatedService'; + SQL + end + + def down + # no-op + + # The records were removed by `up` + end +end diff --git a/db/post_migrate/20200511145545_change_variable_interpolation_format_in_common_metrics.rb b/db/post_migrate/20200511145545_change_variable_interpolation_format_in_common_metrics.rb new file mode 100644 index 00000000000..ac3c545350d --- /dev/null +++ b/db/post_migrate/20200511145545_change_variable_interpolation_format_in_common_metrics.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class ChangeVariableInterpolationFormatInCommonMetrics < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + ::Gitlab::DatabaseImporters::CommonMetrics::Importer.new.execute + end + + def down + # no-op + # The import cannot be reversed since we do not know the state that the + # common metrics in the PrometheusMetric table were in before the import. + end +end diff --git a/db/post_migrate/20200511220023_validate_projects_foreign_key_to_namespaces.rb b/db/post_migrate/20200511220023_validate_projects_foreign_key_to_namespaces.rb new file mode 100644 index 00000000000..37a761507fc --- /dev/null +++ b/db/post_migrate/20200511220023_validate_projects_foreign_key_to_namespaces.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class ValidateProjectsForeignKeyToNamespaces < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + FK_NAME = 'fk_projects_namespace_id' + + def up + # Validate the FK added with 20200511080113_add_projects_foreign_key_to_namespaces.rb + validate_foreign_key :projects, :namespace_id, name: FK_NAME + end + + def down + # no-op: No need to invalidate the foreign key + # The inconsistent data are permanently fixed with the data migration + # `20200511083541_cleanup_projects_with_missing_namespace.rb` + # even if it is rolled back. + # If there is an issue with the FK, we'll roll back the migration that adds the FK + end +end diff --git a/db/post_migrate/20200513171959_enable_hashed_storage.rb b/db/post_migrate/20200513171959_enable_hashed_storage.rb new file mode 100644 index 00000000000..53e52b1caff --- /dev/null +++ b/db/post_migrate/20200513171959_enable_hashed_storage.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class EnableHashedStorage < ActiveRecord::Migration[6.0] + DOWNTIME = false + + class ApplicationSetting < ActiveRecord::Base + self.table_name = 'application_settings' + end + + def up + ApplicationSetting.update_all(hashed_storage_enabled: true) + end + + def down + # in 13.0 we are forcing hashed storage to always be enabled for new projects + end +end diff --git a/db/post_migrate/20200514000009_add_not_null_constraint_on_file_store_to_lfs_objects.rb b/db/post_migrate/20200514000009_add_not_null_constraint_on_file_store_to_lfs_objects.rb new file mode 100644 index 00000000000..6b3b9a3155d --- /dev/null +++ b/db/post_migrate/20200514000009_add_not_null_constraint_on_file_store_to_lfs_objects.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddNotNullConstraintOnFileStoreToLfsObjects < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_not_null_constraint(:lfs_objects, :file_store, validate: false) + end + + def down + remove_not_null_constraint(:lfs_objects, :file_store) + end +end diff --git a/db/post_migrate/20200514000132_add_not_null_constraint_on_store_to_uploads.rb b/db/post_migrate/20200514000132_add_not_null_constraint_on_store_to_uploads.rb new file mode 100644 index 00000000000..c5f1cfa79b8 --- /dev/null +++ b/db/post_migrate/20200514000132_add_not_null_constraint_on_store_to_uploads.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddNotNullConstraintOnStoreToUploads < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_not_null_constraint(:uploads, :store, validate: false) + end + + def down + remove_not_null_constraint(:uploads, :store) + end +end diff --git a/db/post_migrate/20200514000340_add_not_null_constraint_on_file_store_to_ci_jobs_artifacts.rb b/db/post_migrate/20200514000340_add_not_null_constraint_on_file_store_to_ci_jobs_artifacts.rb new file mode 100644 index 00000000000..5759803e3b7 --- /dev/null +++ b/db/post_migrate/20200514000340_add_not_null_constraint_on_file_store_to_ci_jobs_artifacts.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddNotNullConstraintOnFileStoreToCiJobsArtifacts < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_not_null_constraint(:ci_job_artifacts, :file_store, validate: false) + end + + def down + remove_not_null_constraint(:ci_job_artifacts, :file_store) + end +end |