Add latest changes from gitlab-org/gitlab@master

9 files changed, 185 insertions, 2 deletions

diff --git a/db/migrate/20201104204739_create_bulk_import_trackers.rb b/db/migrate/20201104204739_create_bulk_import_trackers.rb
new file mode 100644
index 00000000000..906bd06248d
--- /dev/null
+++ b/db/migrate/20201104204739_create_bulk_import_trackers.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class CreateBulkImportTrackers < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    with_lock_retries do
+      unless table_exists?(:bulk_import_trackers)
+        create_table :bulk_import_trackers do |t|
+          t.references :bulk_import_entity,
+            null: false,
+            index: false,
+            foreign_key: { on_delete: :cascade }
+
+          t.text :relation, null: false
+          t.text :next_page
+          t.boolean :has_next_page, default: false, null: false
+
+          t.index %w(bulk_import_entity_id relation),
+            unique: true,
+            name: :bulk_import_trackers_uniq_relation_by_entity
+        end
+      end
+    end
+
+    add_check_constraint :bulk_import_trackers,
+      '(has_next_page IS FALSE or next_page IS NOT NULL)',
+      'check_next_page_requirement'
+    add_text_limit :bulk_import_trackers, :relation, 255
+    add_text_limit :bulk_import_trackers, :next_page, 255
+  end
+
+  def down
+    with_lock_retries do
+      drop_table :bulk_import_trackers
+    end
+  end
+end
diff --git a/db/migrate/20201110133629_change_index_mr_metrics_target_project_id.rb b/db/migrate/20201110133629_change_index_mr_metrics_target_project_id.rb
new file mode 100644
index 00000000000..5bd810207d8
--- /dev/null
+++ b/db/migrate/20201110133629_change_index_mr_metrics_target_project_id.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class ChangeIndexMrMetricsTargetProjectId < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+  INDEX_NAME = 'index_merge_request_metrics_on_target_project_id_merged_at'
+  NULLS_LAST_INDEX_NAME = 'index_mr_metrics_on_target_project_id_merged_at_nulls_last'
+
+  def up
+    add_concurrent_index :merge_request_metrics, [:target_project_id, :merged_at, :id], order: { merged_at: 'DESC NULLS LAST', id: 'DESC' }, name: NULLS_LAST_INDEX_NAME
+    remove_concurrent_index_by_name(:merge_request_metrics, INDEX_NAME)
+  end
+
+  def down
+    add_concurrent_index :merge_request_metrics, [:target_project_id, :merged_at], name: INDEX_NAME
+    remove_concurrent_index_by_name(:merge_request_metrics, NULLS_LAST_INDEX_NAME)
+  end
+end
diff --git a/db/migrate/20201111152859_add_missing_expression_indexes.rb b/db/migrate/20201111152859_add_missing_expression_indexes.rb
new file mode 100644
index 00000000000..e2742f7f3bc
--- /dev/null
+++ b/db/migrate/20201111152859_add_missing_expression_indexes.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+class AddMissingExpressionIndexes < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  INDEXES = [
+    [:namespaces, :index_on_namespaces_lower_name, 'LOWER(name)'],
+    [:namespaces, :index_on_namespaces_lower_path, 'LOWER(path)'],
+    [:projects, :index_on_projects_lower_path, 'LOWER(path)'],
+    [:routes, :index_on_routes_lower_path, 'LOWER(path)'],
+    [:users, :index_on_users_lower_username, 'LOWER(username)'],
+    [:users, :index_on_users_lower_email, 'LOWER(email)']
+  ]
+
+  def up
+    # Those indexes had been introduced before, but they haven't been
+    # captured in structure.sql. For installations that already have it,
+    # this is a no-op - others will get it retroactively with
+    # this migration.
+
+    tables = Set.new
+
+    INDEXES.each do |(table, name, expression)|
+      unless index_name_exists?(table, name)
+        add_concurrent_index table, expression, name: name
+        tables.add(table)
+      end
+    end
+
+    # Rebuild statistics on affected tables only
+    tables.each do |table|
+      execute("ANALYZE #{table}")
+    end
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/post_migrate/20201103110018_schedule_merge_request_cleanup_schedules_backfill.rb b/db/post_migrate/20201103110018_schedule_merge_request_cleanup_schedules_backfill.rb
new file mode 100644
index 00000000000..77057205b09
--- /dev/null
+++ b/db/post_migrate/20201103110018_schedule_merge_request_cleanup_schedules_backfill.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+class ScheduleMergeRequestCleanupSchedulesBackfill < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MIGRATION = 'BackfillMergeRequestCleanupSchedules'
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 10_000
+  TEMP_INDEX_NAME = 'merge_requests_state_id_temp_index'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :merge_requests, :id, name: TEMP_INDEX_NAME, where: "state_id IN (2, 3)"
+
+    eligible_mrs = Gitlab::BackgroundMigration::BackfillMergeRequestCleanupSchedules::MergeRequest.eligible
+
+    queue_background_migration_jobs_by_range_at_intervals(
+      eligible_mrs,
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE
+    )
+  end
+
+  def down
+    remove_concurrent_index_by_name :merge_requests, TEMP_INDEX_NAME
+  end
+end
diff --git a/db/schema_migrations/20201103110018 b/db/schema_migrations/20201103110018
new file mode 100644
index 00000000000..82ab2fda543
--- /dev/null
+++ b/db/schema_migrations/20201103110018
@@ -0,0 +1 @@
+8a45a6186d7e18f1dea43593dc0226860fc2e8d3ae386f50a022958b758c7c75
\ No newline at end of file
diff --git a/db/schema_migrations/20201104204739 b/db/schema_migrations/20201104204739
new file mode 100644
index 00000000000..83794fd569f
--- /dev/null
+++ b/db/schema_migrations/20201104204739
@@ -0,0 +1 @@
+9431c771b14d61851e8e69b3a789f222463bbe460078a35c8ad3cbcf8df8b077
\ No newline at end of file
diff --git a/db/schema_migrations/20201110133629 b/db/schema_migrations/20201110133629
new file mode 100644
index 00000000000..487b5eab7c1
--- /dev/null
+++ b/db/schema_migrations/20201110133629
@@ -0,0 +1 @@
+83773b825db9b2671fd4ffb2c0d6733036737385ce7a933040011026b34ba1e1
\ No newline at end of file
diff --git a/db/schema_migrations/20201111152859 b/db/schema_migrations/20201111152859
new file mode 100644
index 00000000000..2d5b337b857
--- /dev/null
+++ b/db/schema_migrations/20201111152859
@@ -0,0 +1 @@
+4c5baa6a09a339fac544f830d5ef822b1e7e4eae8431bd91df5113125accbc77
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 0d913c08f08..a7334cd10f2 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -9918,6 +9918,26 @@ CREATE SEQUENCE bulk_import_entities_id_seq
 
 ALTER SEQUENCE bulk_import_entities_id_seq OWNED BY bulk_import_entities.id;
 
+CREATE TABLE bulk_import_trackers (
+    id bigint NOT NULL,
+    bulk_import_entity_id bigint NOT NULL,
+    relation text NOT NULL,
+    next_page text,
+    has_next_page boolean DEFAULT false NOT NULL,
+    CONSTRAINT check_2d45cae629 CHECK ((char_length(relation) <= 255)),
+    CONSTRAINT check_40aeaa600b CHECK ((char_length(next_page) <= 255)),
+    CONSTRAINT check_next_page_requirement CHECK (((has_next_page IS FALSE) OR (next_page IS NOT NULL)))
+);
+
+CREATE SEQUENCE bulk_import_trackers_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE bulk_import_trackers_id_seq OWNED BY bulk_import_trackers.id;
+
 CREATE TABLE bulk_imports (
     id bigint NOT NULL,
     user_id integer NOT NULL,
@@ -17621,6 +17641,8 @@ ALTER TABLE ONLY bulk_import_configurations ALTER COLUMN id SET DEFAULT nextval(
 
 ALTER TABLE ONLY bulk_import_entities ALTER COLUMN id SET DEFAULT nextval('bulk_import_entities_id_seq'::regclass);
 
+ALTER TABLE ONLY bulk_import_trackers ALTER COLUMN id SET DEFAULT nextval('bulk_import_trackers_id_seq'::regclass);
+
 ALTER TABLE ONLY bulk_imports ALTER COLUMN id SET DEFAULT nextval('bulk_imports_id_seq'::regclass);
 
 ALTER TABLE ONLY chat_names ALTER COLUMN id SET DEFAULT nextval('chat_names_id_seq'::regclass);
@@ -18631,6 +18653,9 @@ ALTER TABLE ONLY bulk_import_configurations
 ALTER TABLE ONLY bulk_import_entities
     ADD CONSTRAINT bulk_import_entities_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY bulk_import_trackers
+    ADD CONSTRAINT bulk_import_trackers_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY bulk_imports
     ADD CONSTRAINT bulk_imports_pkey PRIMARY KEY (id);
 
@@ -19914,6 +19939,8 @@ CREATE INDEX backup_labels_title_idx ON backup_labels USING btree (title);
 
 CREATE INDEX backup_labels_type_project_id_idx ON backup_labels USING btree (type, project_id);
 
+CREATE UNIQUE INDEX bulk_import_trackers_uniq_relation_by_entity ON bulk_import_trackers USING btree (bulk_import_entity_id, relation);
+
 CREATE INDEX ci_builds_gitlab_monitor_metrics ON ci_builds USING btree (status, created_at, project_id) WHERE ((type)::text = 'Ci::Build'::text);
 
 CREATE INDEX code_owner_approval_required ON protected_branches USING btree (project_id, code_owner_approval_required) WHERE (code_owner_approval_required = true);
@@ -21136,8 +21163,6 @@ CREATE INDEX index_merge_request_metrics_on_pipeline_id ON merge_request_metrics
 
 CREATE INDEX index_merge_request_metrics_on_target_project_id ON merge_request_metrics USING btree (target_project_id);
 
-CREATE INDEX index_merge_request_metrics_on_target_project_id_merged_at ON merge_request_metrics USING btree (target_project_id, merged_at);
-
 CREATE UNIQUE INDEX index_merge_request_reviewers_on_merge_request_id_and_user_id ON merge_request_reviewers USING btree (merge_request_id, user_id);
 
 CREATE INDEX index_merge_request_reviewers_on_user_id ON merge_request_reviewers USING btree (user_id);
@@ -21228,6 +21253,8 @@ CREATE INDEX index_mr_cleanup_schedules_timestamps ON merge_request_cleanup_sche
 
 CREATE UNIQUE INDEX index_mr_context_commits_on_merge_request_id_and_sha ON merge_request_context_commits USING btree (merge_request_id, sha);
 
+CREATE INDEX index_mr_metrics_on_target_project_id_merged_at_nulls_last ON merge_request_metrics USING btree (target_project_id, merged_at DESC NULLS LAST, id DESC);
+
 CREATE UNIQUE INDEX index_namespace_aggregation_schedules_on_namespace_id ON namespace_aggregation_schedules USING btree (namespace_id);
 
 CREATE UNIQUE INDEX index_namespace_root_storage_statistics_on_namespace_id ON namespace_root_storage_statistics USING btree (namespace_id);
@@ -21326,12 +21353,24 @@ CREATE UNIQUE INDEX index_on_instance_statistics_recorded_at_and_identifier ON a
 
 CREATE INDEX index_on_label_links_all_columns ON label_links USING btree (target_id, label_id, target_type);
 
+CREATE INDEX index_on_namespaces_lower_name ON namespaces USING btree (lower((name)::text));
+
+CREATE INDEX index_on_namespaces_lower_path ON namespaces USING btree (lower((path)::text));
+
+CREATE INDEX index_on_projects_lower_path ON projects USING btree (lower((path)::text));
+
+CREATE INDEX index_on_routes_lower_path ON routes USING btree (lower((path)::text));
+
 CREATE UNIQUE INDEX index_on_segment_selections_group_id_segment_id ON analytics_devops_adoption_segment_selections USING btree (group_id, segment_id);
 
 CREATE UNIQUE INDEX index_on_segment_selections_project_id_segment_id ON analytics_devops_adoption_segment_selections USING btree (project_id, segment_id);
 
 CREATE INDEX index_on_segment_selections_segment_id ON analytics_devops_adoption_segment_selections USING btree (segment_id);
 
+CREATE INDEX index_on_users_lower_email ON users USING btree (lower((email)::text));
+
+CREATE INDEX index_on_users_lower_username ON users USING btree (lower((username)::text));
+
 CREATE INDEX index_on_users_name_lower ON users USING btree (lower((name)::text));
 
 CREATE INDEX index_open_project_tracker_data_on_service_id ON open_project_tracker_data USING btree (service_id);
@@ -22248,6 +22287,8 @@ CREATE UNIQUE INDEX merge_request_user_mentions_on_mr_id_and_note_id_index ON me
 
 CREATE UNIQUE INDEX merge_request_user_mentions_on_mr_id_index ON merge_request_user_mentions USING btree (merge_request_id) WHERE (note_id IS NULL);
 
+CREATE INDEX merge_requests_state_id_temp_index ON merge_requests USING btree (id) WHERE (state_id = ANY (ARRAY[2, 3]));
+
 CREATE INDEX note_mentions_temp_index ON notes USING btree (id, noteable_type) WHERE (note ~~ '%@%'::text);
 
 CREATE UNIQUE INDEX one_canonical_wiki_page_slug_per_metadata ON wiki_page_slugs USING btree (wiki_page_meta_id) WHERE (canonical = true);
@@ -24122,6 +24163,9 @@ ALTER TABLE ONLY analytics_cycle_analytics_group_stages
 ALTER TABLE ONLY metrics_dashboard_annotations
     ADD CONSTRAINT fk_rails_aeb11a7643 FOREIGN KEY (environment_id) REFERENCES environments(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY bulk_import_trackers
+    ADD CONSTRAINT fk_rails_aed566d3f3 FOREIGN KEY (bulk_import_entity_id) REFERENCES bulk_import_entities(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY pool_repositories
     ADD CONSTRAINT fk_rails_af3f8c5d62 FOREIGN KEY (shard_id) REFERENCES shards(id) ON DELETE RESTRICT;