Add latest changes from gitlab-org/gitlab@13-8-stable-eev13.8.0-rc42

17 files changed, 352 insertions, 10 deletions

diff --git a/db/post_migrate/20190402224749_schedule_merge_request_assignees_migration_progress_check.rb b/db/post_migrate/20190402224749_schedule_merge_request_assignees_migration_progress_check.rb
index 6fb67deb834..8ec6a4a24ec 100644
--- a/db/post_migrate/20190402224749_schedule_merge_request_assignees_migration_progress_check.rb
+++ b/db/post_migrate/20190402224749_schedule_merge_request_assignees_migration_progress_check.rb
@@ -3,7 +3,7 @@
 class ScheduleMergeRequestAssigneesMigrationProgressCheck < ActiveRecord::Migration[5.0]
   include Gitlab::Database::MigrationHelpers
 
-  MIGRATION = 'MergeRequestAssigneesMigrationProgressCheck'.freeze
+  MIGRATION = 'MergeRequestAssigneesMigrationProgressCheck'
 
   DOWNTIME = false
 
diff --git a/db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb b/db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb
index cee99f7cf2e..845c855358b 100644
--- a/db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb
+++ b/db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb
@@ -20,8 +20,8 @@ class ScheduleSyncIssuablesStateIdWhereNil < ActiveRecord::Migration[5.1]
   #
   BATCH_SIZE = 5000
   DELAY_INTERVAL = 120.seconds.to_i
-  ISSUES_MIGRATION = 'SyncIssuesStateId'.freeze
-  MERGE_REQUESTS_MIGRATION = 'SyncMergeRequestsStateId'.freeze
+  ISSUES_MIGRATION = 'SyncIssuesStateId'
+  MERGE_REQUESTS_MIGRATION = 'SyncMergeRequestsStateId'
 
   disable_ddl_transaction!
 
diff --git a/db/post_migrate/20190611161642_add_index_to_events_and_audit_events_created_at_author_id.rb b/db/post_migrate/20190611161642_add_index_to_events_and_audit_events_created_at_author_id.rb
index 342e83d6322..5b9afbe128c 100644
--- a/db/post_migrate/20190611161642_add_index_to_events_and_audit_events_created_at_author_id.rb
+++ b/db/post_migrate/20190611161642_add_index_to_events_and_audit_events_created_at_author_id.rb
@@ -7,9 +7,9 @@ class AddIndexToEventsAndAuditEventsCreatedAtAuthorId < ActiveRecord::Migration[
   include Gitlab::Database::MigrationHelpers
 
   DOWNTIME = false
-  INDEX_NAME = 'analytics_index_%s_on_created_at_and_author_id'.freeze
-  EVENTS_INDEX_NAME = (INDEX_NAME % 'events').freeze
-  AUDIT_EVENTS_INDEX_NAME = (INDEX_NAME % 'audit_events').freeze
+  INDEX_NAME = 'analytics_index_%s_on_created_at_and_author_id'
+  EVENTS_INDEX_NAME = (INDEX_NAME % 'events')
+  AUDIT_EVENTS_INDEX_NAME = (INDEX_NAME % 'audit_events')
 
   disable_ddl_transaction!
 
diff --git a/db/post_migrate/20191030223057_backfill_version_author_and_created_at.rb b/db/post_migrate/20191030223057_backfill_version_author_and_created_at.rb
index 7a6d0b089a0..5fcec83bfc3 100644
--- a/db/post_migrate/20191030223057_backfill_version_author_and_created_at.rb
+++ b/db/post_migrate/20191030223057_backfill_version_author_and_created_at.rb
@@ -2,7 +2,7 @@
 
 class BackfillVersionAuthorAndCreatedAt < ActiveRecord::Migration[5.2]
   DOWNTIME = false
-  MIGRATION = 'BackfillVersionDataFromGitaly'.freeze
+  MIGRATION = 'BackfillVersionDataFromGitaly'
   BATCH_SIZE = 500
 
   disable_ddl_transaction!
diff --git a/db/post_migrate/20191104142124_nullify_users_role.rb b/db/post_migrate/20191104142124_nullify_users_role.rb
index 540d58e95ba..ab8eae46745 100644
--- a/db/post_migrate/20191104142124_nullify_users_role.rb
+++ b/db/post_migrate/20191104142124_nullify_users_role.rb
@@ -4,7 +4,7 @@ class NullifyUsersRole < ActiveRecord::Migration[5.2]
   include Gitlab::Database::MigrationHelpers
 
   disable_ddl_transaction!
-  INDEX_NAME = 'partial_index_users_updated_at_for_cleaning_mistaken_values'.freeze
+  INDEX_NAME = 'partial_index_users_updated_at_for_cleaning_mistaken_values'
 
   DOWNTIME = false
 
diff --git a/db/post_migrate/20200207185149_schedule_fix_orphan_promoted_issues.rb b/db/post_migrate/20200207185149_schedule_fix_orphan_promoted_issues.rb
index 83ba56501dd..d25d6ed2dc7 100644
--- a/db/post_migrate/20200207185149_schedule_fix_orphan_promoted_issues.rb
+++ b/db/post_migrate/20200207185149_schedule_fix_orphan_promoted_issues.rb
@@ -5,7 +5,7 @@ class ScheduleFixOrphanPromotedIssues < ActiveRecord::Migration[5.2]
 
   DOWNTIME = false
   BATCH_SIZE = 100
-  BACKGROUND_MIGRATION = 'FixOrphanPromotedIssues'.freeze
+  BACKGROUND_MIGRATION = 'FixOrphanPromotedIssues'
 
   disable_ddl_transaction!
 
diff --git a/db/post_migrate/20200217225719_schedule_migrate_security_scans.rb b/db/post_migrate/20200217225719_schedule_migrate_security_scans.rb
index 7ef204ed9de..087c189f9fd 100644
--- a/db/post_migrate/20200217225719_schedule_migrate_security_scans.rb
+++ b/db/post_migrate/20200217225719_schedule_migrate_security_scans.rb
@@ -6,7 +6,7 @@ class ScheduleMigrateSecurityScans < ActiveRecord::Migration[5.2]
   DOWNTIME = false
   INTERVAL = 2.minutes.to_i
   BATCH_SIZE = 10_000
-  MIGRATION = 'MigrateSecurityScans'.freeze
+  MIGRATION = 'MigrateSecurityScans'
 
   disable_ddl_transaction!
 
diff --git a/db/post_migrate/20200716234259_remove_duplicate_labels_from_group.rb b/db/post_migrate/20200716234259_remove_duplicate_labels_from_group.rb
new file mode 100644
index 00000000000..f19a209092b
--- /dev/null
+++ b/db/post_migrate/20200716234259_remove_duplicate_labels_from_group.rb
@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+class RemoveDuplicateLabelsFromGroup < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  CREATE = 1
+  RENAME = 2
+
+  disable_ddl_transaction!
+
+  class BackupLabel < ApplicationRecord
+    include EachBatch
+
+    self.table_name = 'backup_labels'
+  end
+
+  class Label < ApplicationRecord
+    self.table_name = 'labels'
+  end
+
+  class Group < ApplicationRecord
+    include EachBatch
+
+    self.table_name = 'namespaces'
+  end
+
+  BATCH_SIZE = 10_000
+
+  def up
+    # Split to smaller chunks
+    # Loop rather than background job, every 10,000
+    # there are ~1,800,000 groups in total (excluding personal namespaces, which can't have labels)
+    Group.where(type: 'Group').each_batch(of: BATCH_SIZE) do |batch|
+      range = batch.pluck('MIN(id)', 'MAX(id)').first
+
+      transaction do
+        remove_full_duplicates(*range)
+      end
+
+      transaction do
+        rename_partial_duplicates(*range)
+      end
+    end
+  end
+
+  DOWN_BATCH_SIZE = 1000
+
+  def down
+    BackupLabel.where('project_id IS NULL AND group_id IS NOT NULL').each_batch(of: DOWN_BATCH_SIZE) do |batch|
+      range = batch.pluck('MIN(id)', 'MAX(id)').first
+
+      restore_renamed_labels(*range)
+      restore_deleted_labels(*range)
+    end
+  end
+
+  def remove_full_duplicates(start_id, stop_id)
+    # Fields that are considered duplicate:
+    # group_id title template description type color
+
+    duplicate_labels = ApplicationRecord.connection.execute(<<-SQL.squish)
+WITH data AS (
+  SELECT labels.*,
+  row_number() OVER (PARTITION BY labels.group_id, labels.title, labels.template, labels.description, labels.type, labels.color ORDER BY labels.id) AS row_number,
+  #{CREATE} AS restore_action
+  FROM labels
+  WHERE labels.group_id BETWEEN #{start_id} AND #{stop_id}
+  AND NOT EXISTS (SELECT * FROM board_labels WHERE board_labels.label_id = labels.id)
+  AND NOT EXISTS (SELECT * FROM label_links WHERE label_links.label_id = labels.id)
+  AND NOT EXISTS (SELECT * FROM label_priorities WHERE label_priorities.label_id = labels.id)
+  AND NOT EXISTS (SELECT * FROM lists WHERE lists.label_id = labels.id)
+  AND NOT EXISTS (SELECT * FROM resource_label_events WHERE resource_label_events.label_id = labels.id)
+) SELECT * FROM data WHERE row_number > 1;
+    SQL
+
+    if duplicate_labels.any?
+      # create backup records
+      BackupLabel.insert_all!(duplicate_labels.map { |label| label.except("row_number") })
+
+      Label.unscoped.where(id: duplicate_labels.pluck("id")).delete_all
+    end
+  end
+
+  def rename_partial_duplicates(start_id, stop_id)
+    # We need to ensure that the new title (with `_duplicate#{ID}`) doesn't exceed the limit.
+    # Truncate the original title (if needed) to 245 characters minus the length of the ID
+    # then add `_duplicate#{ID}`
+
+    soft_duplicates = ApplicationRecord.connection.execute(<<-SQL.squish)
+WITH data AS (
+  SELECT
+     *,
+     substring(title from 1 for 245 - length(id::text)) || '_duplicate' || id::text as new_title,
+     #{RENAME} AS restore_action,
+     row_number() OVER (PARTITION BY group_id, title ORDER BY id) AS row_number
+  FROM labels
+  WHERE group_id BETWEEN #{start_id} AND #{stop_id}
+) SELECT * FROM data WHERE row_number > 1;
+    SQL
+
+    if soft_duplicates.any?
+      # create backup records
+      BackupLabel.insert_all!(soft_duplicates.map { |label| label.except("row_number") })
+
+      ApplicationRecord.connection.execute(<<-SQL.squish)
+UPDATE labels SET title = substring(title from 1 for 245 - length(id::text)) || '_duplicate' || id::text
+WHERE labels.id IN (#{soft_duplicates.map { |dup| dup["id"] }.join(", ")});
+      SQL
+    end
+  end
+
+  def restore_renamed_labels(start_id, stop_id)
+    # the backup label IDs are not incremental, they are copied directly from the Labels table
+    ApplicationRecord.connection.execute(<<-SQL.squish)
+WITH backups AS (
+  SELECT id, title
+  FROM backup_labels
+  WHERE id BETWEEN #{start_id} AND #{stop_id}
+  AND restore_action = #{RENAME}
+) UPDATE labels SET title = backups.title
+FROM backups
+WHERE labels.id = backups.id;
+    SQL
+  end
+
+  def restore_deleted_labels(start_id, stop_id)
+    ActiveRecord::Base.connection.execute(<<-SQL.squish)
+INSERT INTO labels
+SELECT id, title, color, group_id, created_at, updated_at, template, description, description_html, type, cached_markdown_version FROM backup_labels
+  WHERE backup_labels.id BETWEEN #{start_id} AND #{stop_id}
+  AND backup_labels.project_id IS NULL AND backup_labels.group_id IS NOT NULL
+  AND backup_labels.restore_action = #{CREATE}
+    SQL
+  end
+end
diff --git a/db/post_migrate/20200716234518_add_uniqueness_index_to_label_title_and_group.rb b/db/post_migrate/20200716234518_add_uniqueness_index_to_label_title_and_group.rb
new file mode 100644
index 00000000000..08bab811c00
--- /dev/null
+++ b/db/post_migrate/20200716234518_add_uniqueness_index_to_label_title_and_group.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class AddUniquenessIndexToLabelTitleAndGroup < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  GROUP_AND_TITLE = [:group_id, :title]
+
+  def up
+    add_concurrent_index :labels, GROUP_AND_TITLE, where: "labels.project_id IS NULL", unique: true, name: "index_labels_on_group_id_and_title_unique"
+    remove_concurrent_index :labels, GROUP_AND_TITLE, name: "index_labels_on_group_id_and_title"
+  end
+
+  def down
+    add_concurrent_index :labels, GROUP_AND_TITLE, where: "labels.project_id IS NULL", unique: false, name: "index_labels_on_group_id_and_title"
+    remove_concurrent_index :labels, GROUP_AND_TITLE, name: "index_labels_on_group_id_and_title_unique"
+  end
+end
diff --git a/db/post_migrate/20201207165956_remove_duplicate_services.rb b/db/post_migrate/20201207165956_remove_duplicate_services.rb
new file mode 100644
index 00000000000..1659b9a2095
--- /dev/null
+++ b/db/post_migrate/20201207165956_remove_duplicate_services.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class RemoveDuplicateServices < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    # noop, replaced by 20210112143418_remove_duplicate_services.rb
+  end
+
+  def down
+  end
+end
diff --git a/db/post_migrate/20201208175117_schedule_backfilling_artifact_expiry_migration.rb b/db/post_migrate/20201208175117_schedule_backfilling_artifact_expiry_migration.rb
new file mode 100644
index 00000000000..f11c0bbe33a
--- /dev/null
+++ b/db/post_migrate/20201208175117_schedule_backfilling_artifact_expiry_migration.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class ScheduleBackfillingArtifactExpiryMigration < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  SWITCH_DATE = Time.utc(2020, 6, 22).freeze
+  INDEX_NAME = 'expired_artifacts_temp_index'.freeze
+  INDEX_CONDITION = "expire_at IS NULL AND created_at < '#{SWITCH_DATE}'"
+
+  disable_ddl_transaction!
+
+  class JobArtifact < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'ci_job_artifacts'
+
+    scope :without_expiry_date, -> { where(expire_at: nil) }
+    scope :before_switch, -> { where('created_at < ?', SWITCH_DATE) }
+  end
+
+  def up
+    # Create temporary index for expired artifacts
+    # Needs to be removed in a later migration
+    add_concurrent_index(:ci_job_artifacts, %i(id created_at), where: INDEX_CONDITION, name: INDEX_NAME)
+
+    queue_background_migration_jobs_by_range_at_intervals(
+      JobArtifact.without_expiry_date.before_switch,
+      ::Gitlab::BackgroundMigration::BackfillArtifactExpiryDate,
+      2.minutes,
+      batch_size: 200_000
+    )
+  end
+
+  def down
+    remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME
+  end
+end
diff --git a/db/post_migrate/20201211090634_schedule_populate_finding_uuid_for_vulnerability_feedback.rb b/db/post_migrate/20201211090634_schedule_populate_finding_uuid_for_vulnerability_feedback.rb
new file mode 100644
index 00000000000..3ecb48dab0f
--- /dev/null
+++ b/db/post_migrate/20201211090634_schedule_populate_finding_uuid_for_vulnerability_feedback.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class SchedulePopulateFindingUuidForVulnerabilityFeedback < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MIGRATION_CLASS = 'PopulateFindingUuidForVulnerabilityFeedback'
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+
+  disable_ddl_transaction!
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      Gitlab::BackgroundMigration::PopulateFindingUuidForVulnerabilityFeedback::VulnerabilityFeedback,
+      MIGRATION_CLASS,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE
+    )
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/post_migrate/20201216185336_add_devops_adoption_snapshot_not_null.rb b/db/post_migrate/20201216185336_add_devops_adoption_snapshot_not_null.rb
new file mode 100644
index 00000000000..1bb3c57f3cd
--- /dev/null
+++ b/db/post_migrate/20201216185336_add_devops_adoption_snapshot_not_null.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class AddDevopsAdoptionSnapshotNotNull < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    with_lock_retries do
+      execute(
+        <<~SQL
+          LOCK TABLE analytics_devops_adoption_snapshots IN ACCESS EXCLUSIVE MODE;
+
+          UPDATE analytics_devops_adoption_snapshots SET end_time = date_trunc('month', recorded_at) - interval '1 millisecond';
+
+          ALTER TABLE analytics_devops_adoption_snapshots ALTER COLUMN end_time SET NOT NULL;
+      SQL
+      )
+    end
+  end
+
+  def down
+    with_lock_retries do
+      execute(<<~SQL)
+        ALTER TABLE analytics_devops_adoption_snapshots ALTER COLUMN end_time DROP NOT NULL;
+      SQL
+    end
+  end
+end
diff --git a/db/post_migrate/20201223012231_reindex_ci_pipelines_on_schedule_id_and_id.rb b/db/post_migrate/20201223012231_reindex_ci_pipelines_on_schedule_id_and_id.rb
new file mode 100644
index 00000000000..9ed1aea911a
--- /dev/null
+++ b/db/post_migrate/20201223012231_reindex_ci_pipelines_on_schedule_id_and_id.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class ReindexCiPipelinesOnScheduleIdAndId < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  OLD_INDEX_NAME = 'index_ci_pipelines_on_pipeline_schedule_id'
+  NEW_INDEX_NAME = 'index_ci_pipelines_on_pipeline_schedule_id_and_id'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :ci_pipelines, [:pipeline_schedule_id, :id], name: NEW_INDEX_NAME
+    remove_concurrent_index_by_name :ci_pipelines, OLD_INDEX_NAME
+  end
+
+  def down
+    add_concurrent_index :ci_pipelines, :pipeline_schedule_id, name: OLD_INDEX_NAME
+    remove_concurrent_index_by_name :ci_pipelines, NEW_INDEX_NAME
+  end
+end
diff --git a/db/post_migrate/20210105103649_delete_column_group_id_on_compliance_framework.rb b/db/post_migrate/20210105103649_delete_column_group_id_on_compliance_framework.rb
new file mode 100644
index 00000000000..b13d2fe2d0a
--- /dev/null
+++ b/db/post_migrate/20210105103649_delete_column_group_id_on_compliance_framework.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class DeleteColumnGroupIdOnComplianceFramework < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    remove_column :compliance_management_frameworks, :group_id, :bigint
+  end
+end
diff --git a/db/post_migrate/20210107194543_remove_alerts_service_records.rb b/db/post_migrate/20210107194543_remove_alerts_service_records.rb
new file mode 100644
index 00000000000..51a2f96ac7f
--- /dev/null
+++ b/db/post_migrate/20210107194543_remove_alerts_service_records.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class RemoveAlertsServiceRecords < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class Service < ActiveRecord::Base
+    self.table_name = 'services'
+  end
+
+  def up
+    Service.delete_by(type: 'AlertsService')
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/post_migrate/20210112143418_remove_duplicate_services2.rb b/db/post_migrate/20210112143418_remove_duplicate_services2.rb
new file mode 100644
index 00000000000..83d92a78473
--- /dev/null
+++ b/db/post_migrate/20210112143418_remove_duplicate_services2.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# This replaces the previous post-deployment migration 20201207165956_remove_duplicate_services_spec.rb,
+# we have to run this again due to a bug in how we were receiving the arguments in the background migration.
+class RemoveDuplicateServices2 < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INTERVAL = 2.minutes
+  BATCH_SIZE = 5_000
+  MIGRATION = 'RemoveDuplicateServices'
+
+  disable_ddl_transaction!
+
+  def up
+    project_ids_with_duplicates = Gitlab::BackgroundMigration::RemoveDuplicateServices::Service.project_ids_with_duplicates
+
+    project_ids_with_duplicates.each_batch(of: BATCH_SIZE, column: :project_id) do |batch, index|
+      migrate_in(
+        INTERVAL * index,
+        MIGRATION,
+        batch.pluck(:project_id)
+      )
+    end
+  end
+
+  def down
+  end
+end