Add latest changes from gitlab-org/gitlab@13-0-stable-ee

38 files changed, 827 insertions, 11 deletions

diff --git a/db/post_migrate/20181008200441_remove_circuit_breaker.rb b/db/post_migrate/20181008200441_remove_circuit_breaker.rb
index 378692e8886..09491038e50 100644
--- a/db/post_migrate/20181008200441_remove_circuit_breaker.rb
+++ b/db/post_migrate/20181008200441_remove_circuit_breaker.rb
@@ -24,7 +24,9 @@ class RemoveCircuitBreaker < ActiveRecord::Migration[4.2]
 
   def down
     CIRCUIT_BREAKER_COLUMS_WITH_DEFAULT.each do |column, default|
+      # rubocop:disable Migration/AddColumnWithDefault
       add_column_with_default(:application_settings, column, :integer, default: default) unless column_exists?(:application_settings, column)
+      # rubocop:enable Migration/AddColumnWithDefault
     end
   end
 end
diff --git a/db/post_migrate/20181013005024_remove_koding_from_application_settings.rb b/db/post_migrate/20181013005024_remove_koding_from_application_settings.rb
index b6e5473e896..550ad94f4ab 100644
--- a/db/post_migrate/20181013005024_remove_koding_from_application_settings.rb
+++ b/db/post_migrate/20181013005024_remove_koding_from_application_settings.rb
@@ -12,6 +12,6 @@ class RemoveKodingFromApplicationSettings < ActiveRecord::Migration[4.2]
 
   def down
     add_column :application_settings, :koding_enabled, :boolean # rubocop:disable Migration/SaferBooleanColumn
-    add_column :application_settings, :koding_url, :string # rubocop:disable Migration/AddLimitToStringColumns
+    add_column :application_settings, :koding_url, :string
   end
 end
diff --git a/db/post_migrate/20190404231137_remove_alternate_url_from_geo_nodes.rb b/db/post_migrate/20190404231137_remove_alternate_url_from_geo_nodes.rb
index 8e7ef0ec54f..785ceb2fb28 100644
--- a/db/post_migrate/20190404231137_remove_alternate_url_from_geo_nodes.rb
+++ b/db/post_migrate/20190404231137_remove_alternate_url_from_geo_nodes.rb
@@ -16,6 +16,6 @@ class RemoveAlternateUrlFromGeoNodes < ActiveRecord::Migration[5.0]
   end
 
   def down
-    add_column :geo_nodes, :alternate_url, :string # rubocop:disable Migration/AddLimitToStringColumns
+    add_column :geo_nodes, :alternate_url, :string
   end
 end
diff --git a/db/post_migrate/20190625184066_remove_sentry_from_application_settings.rb b/db/post_migrate/20190625184066_remove_sentry_from_application_settings.rb
index 9d71bfafffb..7a0923aabd8 100644
--- a/db/post_migrate/20190625184066_remove_sentry_from_application_settings.rb
+++ b/db/post_migrate/20190625184066_remove_sentry_from_application_settings.rb
@@ -28,11 +28,13 @@ class RemoveSentryFromApplicationSettings < ActiveRecord::Migration[5.0]
 
   def down
     SENTRY_ENABLED_COLUMNS.each do |column|
+      # rubocop:disable Migration/AddColumnWithDefault
       add_column_with_default(:application_settings, column, :boolean, default: false, allow_null: false) unless column_exists?(:application_settings, column)
+      # rubocop:enable Migration/AddColumnWithDefault
     end
 
     SENTRY_DSN_COLUMNS.each do |column|
-      add_column(:application_settings, column, :string) unless column_exists?(:application_settings, column) # rubocop:disable Migration/AddLimitToStringColumns
+      add_column(:application_settings, column, :string) unless column_exists?(:application_settings, column)
     end
   end
 end
diff --git a/db/post_migrate/20191030193050_remove_pendo_from_application_settings.rb b/db/post_migrate/20191030193050_remove_pendo_from_application_settings.rb
index 33bbe6f8ea7..c1a1cc01aa4 100644
--- a/db/post_migrate/20191030193050_remove_pendo_from_application_settings.rb
+++ b/db/post_migrate/20191030193050_remove_pendo_from_application_settings.rb
@@ -13,7 +13,7 @@ class RemovePendoFromApplicationSettings < ActiveRecord::Migration[5.2]
   end
 
   def down
-    add_column_with_default :application_settings, :pendo_enabled, :boolean, default: false, allow_null: false
+    add_column_with_default :application_settings, :pendo_enabled, :boolean, default: false, allow_null: false # rubocop:disable Migration/AddColumnWithDefault
     add_column :application_settings, :pendo_url, :string, limit: 255
   end
 end
diff --git a/db/post_migrate/20191128162854_drop_project_ci_cd_settings_merge_trains_enabled.rb b/db/post_migrate/20191128162854_drop_project_ci_cd_settings_merge_trains_enabled.rb
index c2e6792e611..7f91d5112b3 100644
--- a/db/post_migrate/20191128162854_drop_project_ci_cd_settings_merge_trains_enabled.rb
+++ b/db/post_migrate/20191128162854_drop_project_ci_cd_settings_merge_trains_enabled.rb
@@ -12,6 +12,8 @@ class DropProjectCiCdSettingsMergeTrainsEnabled < ActiveRecord::Migration[5.2]
   end
 
   def down
+    # rubocop:disable Migration/AddColumnWithDefault
     add_column_with_default :project_ci_cd_settings, :merge_trains_enabled, :boolean, default: false, allow_null: true # rubocop:disable Migration/UpdateLargeTable
+    # rubocop:enable Migration/AddColumnWithDefault
   end
 end
diff --git a/db/post_migrate/20191202031812_drop_operations_feature_flags_clients_token.rb b/db/post_migrate/20191202031812_drop_operations_feature_flags_clients_token.rb
index bda461af7bc..93cef322f02 100644
--- a/db/post_migrate/20191202031812_drop_operations_feature_flags_clients_token.rb
+++ b/db/post_migrate/20191202031812_drop_operations_feature_flags_clients_token.rb
@@ -14,7 +14,7 @@ class DropOperationsFeatureFlagsClientsToken < ActiveRecord::Migration[5.2]
 
   def down
     unless column_exists?(:operations_feature_flags_clients, :token)
-      add_column :operations_feature_flags_clients, :token, :string # rubocop:disable Migration/AddLimitToStringColumns
+      add_column :operations_feature_flags_clients, :token, :string
     end
 
     add_concurrent_index :operations_feature_flags_clients, [:project_id, :token], unique: true,
diff --git a/db/post_migrate/20200120083607_remove_storage_version_column_from_snippets.rb b/db/post_migrate/20200120083607_remove_storage_version_column_from_snippets.rb
index e94dc75e65c..62bb3f46cae 100644
--- a/db/post_migrate/20200120083607_remove_storage_version_column_from_snippets.rb
+++ b/db/post_migrate/20200120083607_remove_storage_version_column_from_snippets.rb
@@ -19,7 +19,7 @@ class RemoveStorageVersionColumnFromSnippets < ActiveRecord::Migration[5.2]
   def down
     return if column_exists?(:snippets, :storage_version)
 
-    add_column_with_default(
+    add_column_with_default( # rubocop:disable Migration/AddColumnWithDefault
       :snippets,
       :storage_version,
       :integer,
diff --git a/db/post_migrate/20200128210353_cleanup_optimistic_locking_nulls.rb b/db/post_migrate/20200128210353_cleanup_optimistic_locking_nulls.rb
index 8bc037c7333..5c172159561 100644
--- a/db/post_migrate/20200128210353_cleanup_optimistic_locking_nulls.rb
+++ b/db/post_migrate/20200128210353_cleanup_optimistic_locking_nulls.rb
@@ -31,7 +31,7 @@ class CleanupOptimisticLockingNulls < ActiveRecord::Migration[5.2]
         'CleanupOptimisticLockingNulls',
         2.minutes,
         batch_size: BATCH_SIZE,
-        other_arguments: [table]
+        other_job_arguments: [table]
       )
     end
   end
diff --git a/db/post_migrate/20200212052620_readd_template_column_to_services.rb b/db/post_migrate/20200212052620_readd_template_column_to_services.rb
index 2b0d26b2ad4..c636cb0a07b 100644
--- a/db/post_migrate/20200212052620_readd_template_column_to_services.rb
+++ b/db/post_migrate/20200212052620_readd_template_column_to_services.rb
@@ -15,7 +15,7 @@ class ReaddTemplateColumnToServices < ActiveRecord::Migration[6.0]
     # to production, so we should be okay to re-add it without worrying
     # about doing a data migration.  If we needed to restore the value
     # of `template`, we would look for entries with `project_id IS NULL`.
-    add_column_with_default :services, :template, :boolean, default: false, allow_null: true
+    add_column_with_default :services, :template, :boolean, default: false, allow_null: true # rubocop:disable Migration/AddColumnWithDefault
   end
   # rubocop:enable Migration/UpdateLargeTable
 
diff --git a/db/post_migrate/20200217210353_cleanup_optimistic_locking_nulls_pt2.rb b/db/post_migrate/20200217210353_cleanup_optimistic_locking_nulls_pt2.rb
new file mode 100644
index 00000000000..1ec3e817e8e
--- /dev/null
+++ b/db/post_migrate/20200217210353_cleanup_optimistic_locking_nulls_pt2.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class CleanupOptimisticLockingNullsPt2 < ActiveRecord::Migration[5.2]
+  def change
+    # no-op: the MR that contained this migration was reverted
+  end
+end
diff --git a/db/post_migrate/20200219193058_remove_state_from_issues.rb b/db/post_migrate/20200219193058_remove_state_from_issues.rb
index ac27a9a9b69..007ba600ce7 100644
--- a/db/post_migrate/20200219193058_remove_state_from_issues.rb
+++ b/db/post_migrate/20200219193058_remove_state_from_issues.rb
@@ -18,7 +18,7 @@ class RemoveStateFromIssues < ActiveRecord::Migration[6.0]
     return if issue_state_column_exists?
 
     with_lock_retries do
-      add_column :issues, :state, :string # rubocop:disable Migration/AddLimitToStringColumns
+      add_column :issues, :state, :string
     end
   end
 
diff --git a/db/post_migrate/20200219193117_remove_state_from_merge_requests.rb b/db/post_migrate/20200219193117_remove_state_from_merge_requests.rb
index c99a732f37b..384a694b549 100644
--- a/db/post_migrate/20200219193117_remove_state_from_merge_requests.rb
+++ b/db/post_migrate/20200219193117_remove_state_from_merge_requests.rb
@@ -18,7 +18,7 @@ class RemoveStateFromMergeRequests < ActiveRecord::Migration[6.0]
     return if merge_requests_state_column_exists?
 
     with_lock_retries do
-      add_column :merge_requests, :state, :string # rubocop:disable Migration/AddLimitToStringColumns
+      add_column :merge_requests, :state, :string
     end
   end
 
diff --git a/db/post_migrate/20200221142216_remove_repository_storage_from_snippets.rb b/db/post_migrate/20200221142216_remove_repository_storage_from_snippets.rb
index fb8721a45b8..f9ef985218b 100644
--- a/db/post_migrate/20200221142216_remove_repository_storage_from_snippets.rb
+++ b/db/post_migrate/20200221142216_remove_repository_storage_from_snippets.rb
@@ -16,7 +16,7 @@ class RemoveRepositoryStorageFromSnippets < ActiveRecord::Migration[6.0]
   def down
     return if column_exists?(:snippets, :repository_storage)
 
-    add_column_with_default(
+    add_column_with_default( # rubocop:disable Migration/AddColumnWithDefault
       :snippets,
       :repository_storage,
       :string,
diff --git a/db/post_migrate/20200310215714_migrate_saml_identities_to_scim_identities.rb b/db/post_migrate/20200310215714_migrate_saml_identities_to_scim_identities.rb
new file mode 100644
index 00000000000..e2ec7b62d31
--- /dev/null
+++ b/db/post_migrate/20200310215714_migrate_saml_identities_to_scim_identities.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class MigrateSamlIdentitiesToScimIdentities < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  class Identity < ActiveRecord::Base
+    self.table_name = 'identities'
+
+    include ::EachBatch
+  end
+
+  def up
+    Identity
+      .joins('INNER JOIN saml_providers ON saml_providers.id = identities.saml_provider_id')
+      .where('saml_providers.group_id IN (SELECT group_id FROM scim_oauth_access_tokens)')
+      .select('identities.extern_uid, identities.user_id, saml_providers.group_id, TRUE AS active,
+              identities.created_at, CURRENT_TIMESTAMP AS updated_at')
+      .each_batch do |batch|
+        data_to_insert = batch.map do |record|
+          record.attributes.extract!("extern_uid", "user_id", "group_id", "active", "created_at", "updated_at")
+        end
+
+        Gitlab::Database.bulk_insert(:scim_identities, data_to_insert, on_conflict: :do_nothing)
+      end
+  end
+
+  def down
+  end
+end
diff --git a/db/post_migrate/20200312134637_backfill_environment_id_on_deployment_merge_requests.rb b/db/post_migrate/20200312134637_backfill_environment_id_on_deployment_merge_requests.rb
new file mode 100644
index 00000000000..77cb1ae8508
--- /dev/null
+++ b/db/post_migrate/20200312134637_backfill_environment_id_on_deployment_merge_requests.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class BackfillEnvironmentIdOnDeploymentMergeRequests < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    # no-op
+
+    # this migration is deleted because there is no foreign key for
+    # deployments.environment_id and this caused a failure upgrading
+    # deployments_merge_requests.environment_id
+    #
+    # Details on the following issues:
+    #  * https://gitlab.com/gitlab-org/gitlab/-/issues/217191
+    #  * https://gitlab.com/gitlab-org/gitlab/-/issues/26229
+  end
+
+  def down
+    # no-op
+
+    # this migration is designed to delete duplicated data
+  end
+end
diff --git a/db/post_migrate/20200401091051_remove_reference_columns_from_resource_milestone_events.rb b/db/post_migrate/20200401091051_remove_reference_columns_from_resource_milestone_events.rb
new file mode 100644
index 00000000000..639ab93cf18
--- /dev/null
+++ b/db/post_migrate/20200401091051_remove_reference_columns_from_resource_milestone_events.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class RemoveReferenceColumnsFromResourceMilestoneEvents < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    remove_column :resource_milestone_events, :reference, :text
+    remove_column :resource_milestone_events, :reference_html, :text
+    remove_column :resource_milestone_events, :cached_markdown_version, :integer
+  end
+end
diff --git a/db/post_migrate/20200403132349_remove_old_index_pages_domains_need_auto_ssl_renewal.rb b/db/post_migrate/20200403132349_remove_old_index_pages_domains_need_auto_ssl_renewal.rb
new file mode 100644
index 00000000000..b36dce188df
--- /dev/null
+++ b/db/post_migrate/20200403132349_remove_old_index_pages_domains_need_auto_ssl_renewal.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class RemoveOldIndexPagesDomainsNeedAutoSslRenewal < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  INDEX_NAME = 'index_pages_domains_need_auto_ssl_renewal'
+
+  disable_ddl_transaction!
+
+  def up
+    remove_concurrent_index(:pages_domains, [:certificate_source, :certificate_valid_not_after],
+                            where: "auto_ssl_enabled = true", name: INDEX_NAME)
+  end
+
+  def down
+    add_concurrent_index(:pages_domains, [:certificate_source, :certificate_valid_not_after],
+                         where: "auto_ssl_enabled = true", name: INDEX_NAME)
+  end
+end
diff --git a/db/post_migrate/20200420094444_backfill_snippet_repositories.rb b/db/post_migrate/20200420094444_backfill_snippet_repositories.rb
new file mode 100644
index 00000000000..452a1a5330f
--- /dev/null
+++ b/db/post_migrate/20200420094444_backfill_snippet_repositories.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+class BackfillSnippetRepositories < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INTERVAL = 3.minutes
+  BATCH_SIZE = 100
+  MIGRATION = 'BackfillSnippetRepositories'
+
+  disable_ddl_transaction!
+
+  class Snippet < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'snippets'
+    self.inheritance_column = :_type_disabled
+  end
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(Snippet,
+                                                          MIGRATION,
+                                                          INTERVAL,
+                                                          batch_size: BATCH_SIZE)
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/post_migrate/20200420162730_remove_additional_application_settings_rows.rb b/db/post_migrate/20200420162730_remove_additional_application_settings_rows.rb
new file mode 100644
index 00000000000..e4a0ec1eb4a
--- /dev/null
+++ b/db/post_migrate/20200420162730_remove_additional_application_settings_rows.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class RemoveAdditionalApplicationSettingsRows < ActiveRecord::Migration[6.0]
+  class ApplicationSetting < ActiveRecord::Base
+    self.table_name = 'application_settings'
+  end
+
+  def up
+    return if ApplicationSetting.count == 1
+
+    execute "DELETE from application_settings WHERE id NOT IN (SELECT MAX(id) FROM application_settings);"
+  end
+
+  def down
+    # no changes
+  end
+end
diff --git a/db/post_migrate/20200424043515_drop_namespaces_plan_id.rb b/db/post_migrate/20200424043515_drop_namespaces_plan_id.rb
new file mode 100644
index 00000000000..16a56b16e5a
--- /dev/null
+++ b/db/post_migrate/20200424043515_drop_namespaces_plan_id.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class DropNamespacesPlanId < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    with_lock_retries do
+      remove_column :namespaces, :plan_id
+    end
+  end
+
+  def down
+    unless column_exists?(:namespaces, :plan_id)
+      with_lock_retries do
+        add_column :namespaces, :plan_id, :integer
+      end
+    end
+
+    add_concurrent_index :namespaces, :plan_id
+    add_concurrent_foreign_key :namespaces, :plans, column: :plan_id, on_delete: :nullify
+  end
+end
diff --git a/db/post_migrate/20200427064130_cleanup_optimistic_locking_nulls_pt2_fixed.rb b/db/post_migrate/20200427064130_cleanup_optimistic_locking_nulls_pt2_fixed.rb
new file mode 100644
index 00000000000..63f85fc7156
--- /dev/null
+++ b/db/post_migrate/20200427064130_cleanup_optimistic_locking_nulls_pt2_fixed.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class CleanupOptimisticLockingNullsPt2Fixed < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  TABLES = %w(ci_stages ci_builds ci_pipelines).freeze
+  BATCH_SIZE = 10_000
+
+  def declare_class(table)
+    Class.new(ActiveRecord::Base) do
+      include EachBatch
+
+      self.table_name = table
+      self.inheritance_column = :_type_disabled # Disable STI
+    end
+  end
+
+  def up
+    last_table_final_delay = 0
+
+    TABLES.each do |table|
+      # cleanup wrong index created in the previous migration, it might be there on staging
+      remove_concurrent_index table.to_sym, :lock_version, where: "lock_version IS NULL"
+
+      add_concurrent_index table.to_sym, :id, where: "lock_version IS NULL", name: "tmp_index_#{table}_lock_version"
+
+      last_table_final_delay = queue_background_migration_jobs_by_range_at_intervals(
+        declare_class(table).where(lock_version: nil),
+        'CleanupOptimisticLockingNulls',
+        2.minutes,
+        batch_size: BATCH_SIZE,
+        other_job_arguments: [table],
+        initial_delay: last_table_final_delay
+      )
+    end
+  end
+
+  def down
+    TABLES.each do |table|
+      remove_concurrent_index table.to_sym, :id, where: "lock_version IS NULL", name: "tmp_index_#{table}_lock_version"
+    end
+  end
+end
diff --git a/db/post_migrate/20200428134356_remove_elastic_experimental_indexer_from_application_settings.rb b/db/post_migrate/20200428134356_remove_elastic_experimental_indexer_from_application_settings.rb
new file mode 100644
index 00000000000..a9baf6fd8e3
--- /dev/null
+++ b/db/post_migrate/20200428134356_remove_elastic_experimental_indexer_from_application_settings.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+class RemoveElasticExperimentalIndexerFromApplicationSettings < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    remove_column :application_settings, :elasticsearch_experimental_indexer, :boolean
+  end
+end
diff --git a/db/post_migrate/20200429002150_cleanup_sprints_state_rename.rb b/db/post_migrate/20200429002150_cleanup_sprints_state_rename.rb
new file mode 100644
index 00000000000..7f67a55a19d
--- /dev/null
+++ b/db/post_migrate/20200429002150_cleanup_sprints_state_rename.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class CleanupSprintsStateRename < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    cleanup_concurrent_column_rename :sprints, :state, :state_enum
+  end
+
+  def down
+    undo_cleanup_concurrent_column_rename :sprints, :state, :state_enum
+  end
+end
diff --git a/db/post_migrate/20200506085748_update_undefined_confidence_from_occurrences.rb b/db/post_migrate/20200506085748_update_undefined_confidence_from_occurrences.rb
new file mode 100644
index 00000000000..06c82ad404b
--- /dev/null
+++ b/db/post_migrate/20200506085748_update_undefined_confidence_from_occurrences.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class UpdateUndefinedConfidenceFromOccurrences < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  INDEX_NAME = 'index_vulnerability_occurrences_on_id_and_confidence_eq_zero'
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+  BATCH_SIZE = 1_000
+  INTERVAL = 2.minutes
+
+  # 286_159 records to be updated on GitLab.com
+  def up
+    # create temporary index for undefined vulnerabilities
+    add_concurrent_index(:vulnerability_occurrences, :id, where: 'confidence = 0', name: INDEX_NAME)
+
+    return unless Gitlab.ee?
+
+    migration = Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel
+    migration_name = migration.to_s.demodulize
+    relation = migration::Occurrence.undefined_confidence
+    queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                          migration_name,
+                                                          INTERVAL,
+                                                          batch_size: BATCH_SIZE)
+  end
+
+  def down
+    # no-op
+    # temporary index is to be dropped in a different migration in an upcoming release
+    remove_concurrent_index(:vulnerability_occurrences, :id, where: 'confidence = 0', name: INDEX_NAME)
+    # This migration can not be reversed because we can not know which records had undefined confidence
+  end
+end
diff --git a/db/post_migrate/20200506125731_cleanup_user_highest_roles_population.rb b/db/post_migrate/20200506125731_cleanup_user_highest_roles_population.rb
new file mode 100644
index 00000000000..5e613228c56
--- /dev/null
+++ b/db/post_migrate/20200506125731_cleanup_user_highest_roles_population.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class CleanupUserHighestRolesPopulation < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'index_for_migrating_user_highest_roles_table'
+
+  disable_ddl_transaction!
+
+  def up
+    Gitlab::BackgroundMigration.steal('PopulateUserHighestRolesTable')
+
+    remove_concurrent_index(:users, :id, name: INDEX_NAME)
+  end
+
+  def down
+    add_concurrent_index(:users,
+                         :id,
+                         where: "state = 'active' AND user_type IS NULL AND bot_type IS NULL AND ghost IS NOT TRUE",
+                         name: INDEX_NAME)
+  end
+end
diff --git a/db/post_migrate/20200506154421_migrate_scim_identities_to_saml_for_new_users.rb b/db/post_migrate/20200506154421_migrate_scim_identities_to_saml_for_new_users.rb
new file mode 100644
index 00000000000..718e788aad7
--- /dev/null
+++ b/db/post_migrate/20200506154421_migrate_scim_identities_to_saml_for_new_users.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class MigrateScimIdentitiesToSamlForNewUsers < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  class ScimIdentity < ActiveRecord::Base
+    self.table_name = 'scim_identities'
+
+    belongs_to :user
+
+    include ::EachBatch
+  end
+
+  class Identity < ActiveRecord::Base
+    self.table_name = 'identities'
+
+    belongs_to :saml_provider
+  end
+
+  def up
+    users_with_saml_provider = Identity.select('user_id').joins(:saml_provider)
+
+    ScimIdentity.each_batch do |relation|
+      identity_records = relation
+        .select("scim_identities.extern_uid, 'group_saml', scim_identities.user_id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, saml_providers.id")
+        .joins(:user)
+        .joins('inner join saml_providers on saml_providers.group_id=scim_identities.group_id')
+        .where("date_trunc('second',scim_identities.created_at) at time zone 'UTC' = date_trunc('second',users.created_at)")
+        .where.not(user_id: users_with_saml_provider)
+
+      execute "insert into identities (extern_uid, provider, user_id, created_at, updated_at, saml_provider_id) #{identity_records.to_sql} on conflict do nothing"
+    end
+  end
+
+  def down
+  end
+end
diff --git a/db/post_migrate/20200508091106_remove_bot_type.rb b/db/post_migrate/20200508091106_remove_bot_type.rb
new file mode 100644
index 00000000000..2afcf5308e7
--- /dev/null
+++ b/db/post_migrate/20200508091106_remove_bot_type.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class RemoveBotType < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    remove_concurrent_index_by_name :users, 'index_users_on_bot_type'
+
+    with_lock_retries do
+      remove_column :users, :bot_type
+    end
+  end
+
+  def down
+    unless column_exists?(:users, :bot_type)
+      with_lock_retries do
+        add_column :users, :bot_type, :integer, limit: 2 # rubocop:disable Migration/AddColumnsToWideTables
+      end
+    end
+
+    execute 'UPDATE users set bot_type = user_type WHERE user_type IN(1,2,3,6)'
+
+    add_concurrent_index :users, :bot_type
+  end
+end
diff --git a/db/post_migrate/20200511080113_add_projects_foreign_key_to_namespaces.rb b/db/post_migrate/20200511080113_add_projects_foreign_key_to_namespaces.rb
new file mode 100644
index 00000000000..a7f67a3b5cd
--- /dev/null
+++ b/db/post_migrate/20200511080113_add_projects_foreign_key_to_namespaces.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class AddProjectsForeignKeyToNamespaces < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  FK_NAME = 'fk_projects_namespace_id'
+
+  def up
+    with_lock_retries do
+      add_foreign_key(
+        :projects,
+        :namespaces,
+        column: :namespace_id,
+        on_delete: :restrict,
+        validate: false,
+        name: FK_NAME
+      )
+    end
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key_if_exists :projects, column: :namespace_id, name: FK_NAME
+    end
+  end
+end
diff --git a/db/post_migrate/20200511083541_cleanup_projects_with_missing_namespace.rb b/db/post_migrate/20200511083541_cleanup_projects_with_missing_namespace.rb
new file mode 100644
index 00000000000..442acfc6d16
--- /dev/null
+++ b/db/post_migrate/20200511083541_cleanup_projects_with_missing_namespace.rb
@@ -0,0 +1,263 @@
+# frozen_string_literal: true
+
+# rubocop:disable Migration/PreventStrings
+
+# This migration cleans up Projects that were orphaned when their namespace was deleted
+# Instead of deleting them, we:
+# - Find (or create) the Ghost User
+# - Create (if not already exists) a `lost-and-found` group owned by the Ghost User
+# - Find orphaned projects --> namespace_id can not be found in namespaces
+# - Move the orphaned projects to the `lost-and-found` group
+#   (while making them private and setting `archived=true`)
+#
+# On GitLab.com (2020-05-11) this migration will update 66 orphaned projects
+class CleanupProjectsWithMissingNamespace < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  VISIBILITY_PRIVATE = 0
+  ACCESS_LEVEL_OWNER = 50
+
+  # The batch size of projects to check in each iteration
+  # We expect the selectivity for orphaned projects to be very low:
+  #  (66 orphaned projects out of a total 13.6M)
+  # so 10K should be a safe choice
+  BATCH_SIZE = 10000
+
+  disable_ddl_transaction!
+
+  class UserDetail < ActiveRecord::Base
+    self.table_name = 'user_details'
+
+    belongs_to :user, class_name: 'CleanupProjectsWithMissingNamespace::User'
+  end
+
+  class User < ActiveRecord::Base
+    self.table_name = 'users'
+
+    LOST_AND_FOUND_GROUP = 'lost-and-found'
+    USER_TYPE_GHOST = 5
+    DEFAULT_PROJECTS_LIMIT = 100000
+
+    default_value_for :admin, false
+    default_value_for :can_create_group, true # we need this to create the group
+    default_value_for :can_create_team, false
+    default_value_for :project_view, :files
+    default_value_for :notified_of_own_activity, false
+    default_value_for :preferred_language, I18n.default_locale
+
+    has_one :user_detail, class_name: 'CleanupProjectsWithMissingNamespace::UserDetail'
+    has_one :namespace, -> { where(type: nil) },
+            foreign_key: :owner_id, inverse_of: :owner, autosave: true,
+            class_name: 'CleanupProjectsWithMissingNamespace::Namespace'
+
+    before_save :ensure_namespace_correct
+    before_save :ensure_bio_is_assigned_to_user_details, if: :bio_changed?
+
+    enum project_view: { readme: 0, activity: 1, files: 2 }
+
+    def ensure_namespace_correct
+      if namespace
+        namespace.path = username if username_changed?
+        namespace.name = name if name_changed?
+      else
+        build_namespace(path: username, name: name)
+      end
+    end
+
+    def ensure_bio_is_assigned_to_user_details
+      return if Feature.disabled?(:migrate_bio_to_user_details, default_enabled: true)
+
+      user_detail.bio = bio.to_s[0...255]
+    end
+
+    def user_detail
+      super.presence || build_user_detail
+    end
+
+    # Return (or create if necessary) the `lost-and-found` group
+    def lost_and_found_group
+      existing_lost_and_found_group || Group.create_unique_group(self, LOST_AND_FOUND_GROUP)
+    end
+
+    def existing_lost_and_found_group
+      # There should only be one Group for User Ghost starting with LOST_AND_FOUND_GROUP
+      Group
+        .joins('INNER JOIN members ON namespaces.id = members.source_id')
+        .where('namespaces.type = ?', 'Group')
+        .where('members.type = ?', 'GroupMember')
+        .where('members.source_type = ?', 'Namespace')
+        .where('members.user_id = ?', self.id)
+        .where('members.requested_at IS NULL')
+        .where('members.access_level = ?', ACCESS_LEVEL_OWNER)
+        .find_by(Group.arel_table[:name].matches("#{LOST_AND_FOUND_GROUP}%"))
+    end
+
+    class << self
+      # Return (or create if necessary) the ghost user
+      def ghost
+        email = 'ghost%s@example.com'
+
+        unique_internal(where(user_type: USER_TYPE_GHOST), 'ghost', email) do |u|
+          u.bio = _('This is a "Ghost User", created to hold all issues authored by users that have since been deleted. This user cannot be removed.')
+          u.name = 'Ghost User'
+        end
+      end
+
+      def unique_internal(scope, username, email_pattern, &block)
+        scope.first || create_unique_internal(scope, username, email_pattern, &block)
+      end
+
+      def create_unique_internal(scope, username, email_pattern, &creation_block)
+        # Since we only want a single one of these in an instance, we use an
+        # exclusive lease to ensure that this block is never run concurrently.
+        lease_key = "user:unique_internal:#{username}"
+        lease = Gitlab::ExclusiveLease.new(lease_key, timeout: 1.minute.to_i)
+
+        until uuid = lease.try_obtain
+          # Keep trying until we obtain the lease. To prevent hammering Redis too
+          # much we'll wait for a bit between retries.
+          sleep(1)
+        end
+
+        # Recheck if the user is already present. One might have been
+        # added between the time we last checked (first line of this method)
+        # and the time we acquired the lock.
+        existing_user = uncached { scope.first }
+        return existing_user if existing_user.present?
+
+        uniquify = Uniquify.new
+
+        username = uniquify.string(username) { |s| User.find_by_username(s) }
+
+        email = uniquify.string(-> (n) { Kernel.sprintf(email_pattern, n) }) do |s|
+          User.find_by_email(s)
+        end
+
+        User.create!(
+          username: username,
+          email: email,
+          user_type: USER_TYPE_GHOST,
+          projects_limit: DEFAULT_PROJECTS_LIMIT,
+          state: :active,
+          &creation_block
+        )
+      ensure
+        Gitlab::ExclusiveLease.cancel(lease_key, uuid)
+      end
+    end
+  end
+
+  class Namespace < ActiveRecord::Base
+    self.table_name = 'namespaces'
+
+    belongs_to :owner, class_name: 'CleanupProjectsWithMissingNamespace::User'
+  end
+
+  class Group < Namespace
+    # Disable STI to allow us to manually set "type = 'Group'"
+    # Otherwise rails forces "type = CleanupProjectsWithMissingNamespace::Group"
+    self.inheritance_column = :_type_disabled
+
+    def self.create_unique_group(user, group_name)
+      # 'lost-and-found' may be already defined, find a unique one
+      group_name = Uniquify.new.string(group_name) do |str|
+        Group.where(parent_id: nil, name: str).exists?
+      end
+
+      group = Group.create!(
+        name: group_name,
+        path: group_name,
+        type: 'Group',
+        description: 'Group to store orphaned projects',
+        visibility_level: VISIBILITY_PRIVATE
+      )
+
+      # No need to create a route for the lost-and-found group
+
+      GroupMember.add_user(group, user, ACCESS_LEVEL_OWNER)
+
+      group
+    end
+  end
+
+  class Member < ActiveRecord::Base
+    self.table_name = 'members'
+  end
+
+  class GroupMember < Member
+    NOTIFICATION_SETTING_GLOBAL = 3
+
+    # Disable STI to allow us to manually set "type = 'GroupMember'"
+    # Otherwise rails forces "type = CleanupProjectsWithMissingNamespace::GroupMember"
+    self.inheritance_column = :_type_disabled
+
+    def self.add_user(source, user, access_level)
+      GroupMember.create!(
+        type: 'GroupMember',
+        source_id: source.id,
+        user_id: user.id,
+        source_type: 'Namespace',
+        access_level: access_level,
+        notification_level: NOTIFICATION_SETTING_GLOBAL
+      )
+    end
+  end
+
+  class Project < ActiveRecord::Base
+    self.table_name = 'projects'
+
+    include ::EachBatch
+
+    def self.without_namespace
+      where(
+        'NOT EXISTS (
+          SELECT 1
+          FROM namespaces
+          WHERE projects.namespace_id = namespaces.id
+        )'
+      )
+    end
+  end
+
+  def up
+    # Reset the column information of all the models that update the database
+    # to ensure the Active Record's knowledge of the table structure is current
+    User.reset_column_information
+    Namespace.reset_column_information
+    Member.reset_column_information
+    Project.reset_column_information
+
+    # Find or Create the ghost user
+    ghost_user = User.ghost
+
+    # Find or Create the `lost-and-found`
+    lost_and_found = ghost_user.lost_and_found_group
+
+    # With BATCH_SIZE=10K and projects.count=13.6M
+    # ~1360 iterations will be run:
+    # - each requires on average ~160ms for relation.without_namespace
+    # - worst case scenario is that 66 of those batches will trigger an update (~200ms each)
+    #   In general, we expect less than 5% (=66/13.6M x 10K) to trigger an update
+    # Expected total run time: ~235 seconds (== 220 seconds + 14 seconds)
+    Project.each_batch(of: BATCH_SIZE) do |relation|
+      relation.without_namespace.update_all <<~SQL
+        namespace_id = #{lost_and_found.id},
+        archived = TRUE,
+        visibility_level = #{VISIBILITY_PRIVATE},
+
+        -- Names are expected to be unique inside their namespace
+        --  (uniqueness validation on namespace_id, name)
+        -- Attach the id to the name and path to make sure that they are unique
+        name = name || '_' || id,
+        path = path || '_' || id
+      SQL
+    end
+  end
+
+  def down
+    # no-op: the original state for those projects was inconsistent
+    # Also, the original namespace_id for each project is lost during the update
+  end
+end
+# rubocop:enable Migration/PreventStrings
diff --git a/db/post_migrate/20200511092714_update_undefined_confidence_from_vulnerabilities.rb b/db/post_migrate/20200511092714_update_undefined_confidence_from_vulnerabilities.rb
new file mode 100644
index 00000000000..d6611ddbd66
--- /dev/null
+++ b/db/post_migrate/20200511092714_update_undefined_confidence_from_vulnerabilities.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class UpdateUndefinedConfidenceFromVulnerabilities < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  INDEX_NAME = 'index_vulnerability_on_id_and_confidence_eq_zero'
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+  BATCH_SIZE = 1_000
+  INTERVAL = 2.minutes
+
+  # 87_602 records to be updated on GitLab.com
+  def up
+    # create temporary index for undefined vulnerabilities
+    add_concurrent_index(:vulnerabilities, :id, where: 'confidence = 0', name: INDEX_NAME)
+
+    return unless Gitlab.ee?
+
+    migration = Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel
+    migration_name = migration.to_s.demodulize
+    relation = migration::Vulnerability.undefined_confidence
+    queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                          migration_name,
+                                                          INTERVAL,
+                                                          batch_size: BATCH_SIZE)
+  end
+
+  def down
+    # no-op
+    # temporary index is to be dropped in a different migration in an upcoming release
+    remove_concurrent_index(:vulnerabilities, :id, where: 'confidence = 0', name: INDEX_NAME)
+    # This migration can not be reversed because we can not know which records had undefined confidence
+  end
+end
diff --git a/db/post_migrate/20200511130130_ensure_deprecated_jenkins_service_records_removal.rb b/db/post_migrate/20200511130130_ensure_deprecated_jenkins_service_records_removal.rb
new file mode 100644
index 00000000000..4c1f29f8e47
--- /dev/null
+++ b/db/post_migrate/20200511130130_ensure_deprecated_jenkins_service_records_removal.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class EnsureDeprecatedJenkinsServiceRecordsRemoval < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def up
+    execute <<~SQL.strip
+      DELETE FROM services WHERE type = 'JenkinsDeprecatedService';
+    SQL
+  end
+
+  def down
+    # no-op
+
+    # The records were removed by `up`
+  end
+end
diff --git a/db/post_migrate/20200511145545_change_variable_interpolation_format_in_common_metrics.rb b/db/post_migrate/20200511145545_change_variable_interpolation_format_in_common_metrics.rb
new file mode 100644
index 00000000000..ac3c545350d
--- /dev/null
+++ b/db/post_migrate/20200511145545_change_variable_interpolation_format_in_common_metrics.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class ChangeVariableInterpolationFormatInCommonMetrics < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def up
+    ::Gitlab::DatabaseImporters::CommonMetrics::Importer.new.execute
+  end
+
+  def down
+    # no-op
+    # The import cannot be reversed since we do not know the state that the
+    # common metrics in the PrometheusMetric table were in before the import.
+  end
+end
diff --git a/db/post_migrate/20200511220023_validate_projects_foreign_key_to_namespaces.rb b/db/post_migrate/20200511220023_validate_projects_foreign_key_to_namespaces.rb
new file mode 100644
index 00000000000..37a761507fc
--- /dev/null
+++ b/db/post_migrate/20200511220023_validate_projects_foreign_key_to_namespaces.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class ValidateProjectsForeignKeyToNamespaces < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  FK_NAME = 'fk_projects_namespace_id'
+
+  def up
+    # Validate the FK added with 20200511080113_add_projects_foreign_key_to_namespaces.rb
+    validate_foreign_key :projects, :namespace_id, name: FK_NAME
+  end
+
+  def down
+    # no-op: No need to invalidate the foreign key
+    # The inconsistent data are permanently fixed with the data migration
+    #  `20200511083541_cleanup_projects_with_missing_namespace.rb`
+    # even if it is rolled back.
+    # If there is an issue with the FK, we'll roll back the migration that adds the FK
+  end
+end
diff --git a/db/post_migrate/20200513171959_enable_hashed_storage.rb b/db/post_migrate/20200513171959_enable_hashed_storage.rb
new file mode 100644
index 00000000000..53e52b1caff
--- /dev/null
+++ b/db/post_migrate/20200513171959_enable_hashed_storage.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class EnableHashedStorage < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  class ApplicationSetting < ActiveRecord::Base
+    self.table_name = 'application_settings'
+  end
+
+  def up
+    ApplicationSetting.update_all(hashed_storage_enabled: true)
+  end
+
+  def down
+    # in 13.0 we are forcing hashed storage to always be enabled for new projects
+  end
+end
diff --git a/db/post_migrate/20200514000009_add_not_null_constraint_on_file_store_to_lfs_objects.rb b/db/post_migrate/20200514000009_add_not_null_constraint_on_file_store_to_lfs_objects.rb
new file mode 100644
index 00000000000..6b3b9a3155d
--- /dev/null
+++ b/db/post_migrate/20200514000009_add_not_null_constraint_on_file_store_to_lfs_objects.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AddNotNullConstraintOnFileStoreToLfsObjects < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_not_null_constraint(:lfs_objects, :file_store, validate: false)
+  end
+
+  def down
+    remove_not_null_constraint(:lfs_objects, :file_store)
+  end
+end
diff --git a/db/post_migrate/20200514000132_add_not_null_constraint_on_store_to_uploads.rb b/db/post_migrate/20200514000132_add_not_null_constraint_on_store_to_uploads.rb
new file mode 100644
index 00000000000..c5f1cfa79b8
--- /dev/null
+++ b/db/post_migrate/20200514000132_add_not_null_constraint_on_store_to_uploads.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AddNotNullConstraintOnStoreToUploads < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_not_null_constraint(:uploads, :store, validate: false)
+  end
+
+  def down
+    remove_not_null_constraint(:uploads, :store)
+  end
+end
diff --git a/db/post_migrate/20200514000340_add_not_null_constraint_on_file_store_to_ci_jobs_artifacts.rb b/db/post_migrate/20200514000340_add_not_null_constraint_on_file_store_to_ci_jobs_artifacts.rb
new file mode 100644
index 00000000000..5759803e3b7
--- /dev/null
+++ b/db/post_migrate/20200514000340_add_not_null_constraint_on_file_store_to_ci_jobs_artifacts.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AddNotNullConstraintOnFileStoreToCiJobsArtifacts < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_not_null_constraint(:ci_job_artifacts, :file_store, validate: false)
+  end
+
+  def down
+    remove_not_null_constraint(:ci_job_artifacts, :file_store)
+  end
+end