Add latest changes from gitlab-org/gitlab@master

7 files changed, 178 insertions, 0 deletions

diff --git a/db/post_migrate/20200127111953_cleanup_empty_snippet_user_mentions.rb b/db/post_migrate/20200127111953_cleanup_empty_snippet_user_mentions.rb
new file mode 100644
index 00000000000..aad688fef3f
--- /dev/null
+++ b/db/post_migrate/20200127111953_cleanup_empty_snippet_user_mentions.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class CleanupEmptySnippetUserMentions < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+  BATCH_SIZE = 10_000
+
+  class SnippetUserMention < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'snippet_user_mentions'
+  end
+
+  def up
+    # cleanup snippet user mentions with no actual mentions,
+    # re https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24586#note_285982468
+    SnippetUserMention
+      .where(mentioned_users_ids: nil)
+      .where(mentioned_groups_ids: nil)
+      .where(mentioned_projects_ids: nil)
+      .each_batch(of: BATCH_SIZE) do |batch|
+      batch.delete_all
+    end
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/post_migrate/20200127131953_migrate_snippet_mentions_to_db.rb b/db/post_migrate/20200127131953_migrate_snippet_mentions_to_db.rb
new file mode 100644
index 00000000000..e25c2c2982a
--- /dev/null
+++ b/db/post_migrate/20200127131953_migrate_snippet_mentions_to_db.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class MigrateSnippetMentionsToDb < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DELAY = 2.minutes.to_i
+  BATCH_SIZE = 10_000
+  MIGRATION = 'UserMentions::CreateResourceUserMention'
+
+  JOIN = "LEFT JOIN snippet_user_mentions on snippets.id = snippet_user_mentions.snippet_id"
+  QUERY_CONDITIONS = "(description like '%@%' OR title like '%@%') AND snippet_user_mentions.snippet_id IS NULL"
+
+  disable_ddl_transaction!
+
+  class Snippet < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'snippets'
+  end
+
+  def up
+    Snippet
+      .joins(JOIN)
+      .where(QUERY_CONDITIONS)
+      .each_batch(of: BATCH_SIZE) do |batch, index|
+      range = batch.pluck(Arel.sql('MIN(snippets.id)'), Arel.sql('MAX(snippets.id)')).first
+      migrate_in(index * DELAY, MIGRATION, ['Snippet', JOIN, QUERY_CONDITIONS, false, *range])
+    end
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/post_migrate/20200127141953_add_temporary_snippet_notes_with_mentions_index.rb b/db/post_migrate/20200127141953_add_temporary_snippet_notes_with_mentions_index.rb
new file mode 100644
index 00000000000..ec9b8b76f6f
--- /dev/null
+++ b/db/post_migrate/20200127141953_add_temporary_snippet_notes_with_mentions_index.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class AddTemporarySnippetNotesWithMentionsIndex < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'snippet_mentions_temp_index'
+  INDEX_CONDITION = "note LIKE '%@%'::text AND notes.noteable_type = 'Snippet'"
+
+  disable_ddl_transaction!
+
+  def up
+    # create temporary index for notes with mentions, may take well over 1h
+    add_concurrent_index(:notes, :id, where: INDEX_CONDITION, name: INDEX_NAME)
+  end
+
+  def down
+    remove_concurrent_index(:notes, :id, where: INDEX_CONDITION, name: INDEX_NAME)
+  end
+end
diff --git a/db/post_migrate/20200127151953_migrate_snippet_notes_mentions_to_db.rb b/db/post_migrate/20200127151953_migrate_snippet_notes_mentions_to_db.rb
new file mode 100644
index 00000000000..3795a96b426
--- /dev/null
+++ b/db/post_migrate/20200127151953_migrate_snippet_notes_mentions_to_db.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class MigrateSnippetNotesMentionsToDb < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DELAY = 2.minutes.to_i
+  BATCH_SIZE = 10_000
+  MIGRATION = 'UserMentions::CreateResourceUserMention'
+
+  INDEX_CONDITION = "note LIKE '%@%'::text AND notes.noteable_type = 'Snippet'"
+  QUERY_CONDITIONS = "#{INDEX_CONDITION} AND snippet_user_mentions.snippet_id IS NULL"
+  JOIN = 'INNER JOIN snippets ON snippets.id = notes.noteable_id LEFT JOIN snippet_user_mentions ON notes.id = snippet_user_mentions.note_id'
+
+  disable_ddl_transaction!
+
+  class Note < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'notes'
+  end
+
+  def up
+    Note
+      .joins(JOIN)
+      .where(QUERY_CONDITIONS)
+      .each_batch(of: BATCH_SIZE) do |batch, index|
+      range = batch.pluck(Arel.sql('MIN(notes.id)'), Arel.sql('MAX(notes.id)')).first
+      migrate_in(index * DELAY, MIGRATION, ['Snippet', JOIN, QUERY_CONDITIONS, true, *range])
+    end
+  end
+
+  def down
+    # no-op
+    # temporary index is to be dropped in a different migration in an upcoming release:
+    # https://gitlab.com/gitlab-org/gitlab/issues/196842
+  end
+end
diff --git a/db/post_migrate/20200217223651_add_index_to_job_artifact_secure_reports.rb b/db/post_migrate/20200217223651_add_index_to_job_artifact_secure_reports.rb
new file mode 100644
index 00000000000..ca297272f8e
--- /dev/null
+++ b/db/post_migrate/20200217223651_add_index_to_job_artifact_secure_reports.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class AddIndexToJobArtifactSecureReports < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'job_artifacts_secure_reports_temp_index'
+  PARTIAL_FILTER = "file_type BETWEEN 5 AND 8"
+
+  disable_ddl_transaction!
+
+  def up
+    # This is a temporary index used for the migration of Security Reports to Security Scans
+    add_concurrent_index(:ci_job_artifacts,
+                         [:id, :file_type, :job_id, :created_at, :updated_at],
+                         name: INDEX_NAME,
+                         where: PARTIAL_FILTER)
+  end
+
+  def down
+    remove_concurrent_index(:ci_job_artifacts,
+                            [:id, :file_type, :job_id, :created_at, :updated_at])
+  end
+end
diff --git a/db/post_migrate/20200217225719_schedule_migrate_security_scans.rb b/db/post_migrate/20200217225719_schedule_migrate_security_scans.rb
new file mode 100644
index 00000000000..7ef204ed9de
--- /dev/null
+++ b/db/post_migrate/20200217225719_schedule_migrate_security_scans.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class ScheduleMigrateSecurityScans < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INTERVAL = 2.minutes.to_i
+  BATCH_SIZE = 10_000
+  MIGRATION = 'MigrateSecurityScans'.freeze
+
+  disable_ddl_transaction!
+
+  class JobArtifact < ActiveRecord::Base
+    include ::EachBatch
+
+    self.table_name = 'ci_job_artifacts'
+
+    scope :security_reports, -> { where('file_type BETWEEN 5 and 8') }
+  end
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(JobArtifact.security_reports,
+                                                          MIGRATION,
+                                                          INTERVAL,
+                                                          batch_size: BATCH_SIZE)
+  end
+
+  def down
+    # intentionally blank
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 623fdcd0542..c6a5e2dd869 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -766,6 +766,7 @@ ActiveRecord::Schema.define(version: 2020_02_26_162723) do
     t.integer "file_location", limit: 2
     t.index ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id"
     t.index ["file_store"], name: "index_ci_job_artifacts_on_file_store"
+    t.index ["id", "file_type", "job_id", "created_at", "updated_at"], name: "job_artifacts_secure_reports_temp_index", where: "((file_type >= 5) AND (file_type <= 8))"
     t.index ["job_id", "file_type"], name: "index_ci_job_artifacts_on_job_id_and_file_type", unique: true
     t.index ["project_id"], name: "index_ci_job_artifacts_on_project_id"
     t.index ["project_id"], name: "index_ci_job_artifacts_on_project_id_for_security_reports", where: "(file_type = ANY (ARRAY[5, 6, 7, 8]))"
@@ -2824,6 +2825,7 @@ ActiveRecord::Schema.define(version: 2020_02_26_162723) do
     t.index ["discussion_id"], name: "index_notes_on_discussion_id"
     t.index ["id"], name: "design_mentions_temp_index", where: "((note ~~ '%@%'::text) AND ((noteable_type)::text = 'DesignManagement::Design'::text))"
     t.index ["id"], name: "epic_mentions_temp_index", where: "((note ~~ '%@%'::text) AND ((noteable_type)::text = 'Epic'::text))"
+    t.index ["id"], name: "snippet_mentions_temp_index", where: "((note ~~ '%@%'::text) AND ((noteable_type)::text = 'Snippet'::text))"
     t.index ["line_code"], name: "index_notes_on_line_code"
     t.index ["note"], name: "index_notes_on_note_trigram", opclass: :gin_trgm_ops, using: :gin
     t.index ["note"], name: "tmp_idx_on_promoted_notes", where: "(((noteable_type)::text = 'Issue'::text) AND (system IS TRUE) AND (note ~~ 'promoted to epic%'::text))"