Merge branch 'ab-50763-persist-index' into 'master'

Specific CI indexes to avoid statement timeout.

Closes #50763

See merge request gitlab-org/gitlab-ce!23188

4 files changed, 60 insertions, 3 deletions

diff --git a/changelogs/unreleased/ab-50763-persist-index.yml b/changelogs/unreleased/ab-50763-persist-index.yml
new file mode 100644
index 00000000000..0cf11923c5a
--- /dev/null
+++ b/changelogs/unreleased/ab-50763-persist-index.yml
@@ -0,0 +1,5 @@
+---
+title: Add indexes to speed up CI query.
+merge_request: 23188
+author:
+type: performance
diff --git a/config/initializers/postgresql_opclasses_support.rb b/config/initializers/postgresql_opclasses_support.rb
index 70b530415f5..b066f3788ec 100644
--- a/config/initializers/postgresql_opclasses_support.rb
+++ b/config/initializers/postgresql_opclasses_support.rb
@@ -130,9 +130,12 @@ module ActiveRecord
               where = inddef.scan(/WHERE (.+)$/).flatten[0]
               using = inddef.scan(/USING (.+?) /).flatten[0].to_sym
               opclasses = Hash[inddef.scan(/\((.+?)\)(?:$| WHERE )/).flatten[0].split(',').map do |column_and_opclass|
-                                 column, opclass = column_and_opclass.split(' ').map(&:strip)
-                                 [column, opclass] if opclass
-                               end.compact]
+                column, opclass = column_and_opclass.split(' ').map(&:strip)
+              end.reject do |column, opclass|
+                ['desc', 'asc'].include?(opclass&.downcase)
+              end.map do |column, opclass|
+                [column, opclass] if opclass
+              end.compact]
 
               index_attrs = [table_name, index_name, unique, column_names, [], orders, where, nil, using, nil, opclasses]
 
@@ -151,6 +154,9 @@ module ActiveRecord
           def quoted_columns_for_index(column_names, options = {})
             column_opclasses = options[:opclasses] || {}
             column_names.map {|name| "#{quote_column_name(name)} #{column_opclasses[name]}"}
+
+            quoted_columns = Hash[column_names.map { |name| [name.to_sym, "#{quote_column_name(name)} #{column_opclasses[name]}"] }]
+            add_options_for_index_columns(quoted_columns, options).values
           end
       end
     end
diff --git a/db/migrate/20181119132520_add_indexes_to_ci_builds_and_pipelines.rb b/db/migrate/20181119132520_add_indexes_to_ci_builds_and_pipelines.rb
new file mode 100644
index 00000000000..adbc3928b26
--- /dev/null
+++ b/db/migrate/20181119132520_add_indexes_to_ci_builds_and_pipelines.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class AddIndexesToCiBuildsAndPipelines < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    indexes.each do |index|
+      add_concurrent_index(*index)
+    end
+  end
+
+  def down
+    indexes.each do |index|
+      remove_concurrent_index(*index)
+    end
+  end
+
+  private
+
+  def indexes
+    [
+      [
+        :ci_pipelines,
+        [:project_id, :ref, :id],
+        {
+          order: { id: :desc },
+          name: 'index_ci_pipelines_on_project_idandrefandiddesc'
+        }
+      ],
+      [
+        :ci_builds,
+        [:commit_id, :artifacts_expire_at, :id],
+        {
+          where: "type::text = 'Ci::Build'::text AND (retried = false OR retried IS NULL) AND (name::text = ANY (ARRAY['sast'::character varying, 'dependency_scanning'::character varying, 'sast:container'::character varying, 'container_scanning'::character varying, 'dast'::character varying]::text[]))",
+          name: 'index_ci_builds_on_commit_id_and_artifacts_expireatandidpartial'
+        }
+      ]
+    ]
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 8dc1260d177..63267120c58 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -349,6 +349,7 @@ ActiveRecord::Schema.define(version: 20190103140724) do
     t.string "token_encrypted"
     t.index ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree
     t.index ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree
+    t.index ["commit_id", "artifacts_expire_at", "id"], name: "index_ci_builds_on_commit_id_and_artifacts_expireatandidpartial", where: "(((type)::text = 'Ci::Build'::text) AND ((retried = false) OR (retried IS NULL)) AND ((name)::text = ANY (ARRAY[('sast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('sast:container'::character varying)::text, ('container_scanning'::character varying)::text, ('dast'::character varying)::text])))", using: :btree
     t.index ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree
     t.index ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree
     t.index ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree
@@ -483,6 +484,7 @@ ActiveRecord::Schema.define(version: 20190103140724) do
     t.index ["merge_request_id"], name: "index_ci_pipelines_on_merge_request_id", where: "(merge_request_id IS NOT NULL)", using: :btree
     t.index ["pipeline_schedule_id"], name: "index_ci_pipelines_on_pipeline_schedule_id", using: :btree
     t.index ["project_id", "iid"], name: "index_ci_pipelines_on_project_id_and_iid", unique: true, where: "(iid IS NOT NULL)", using: :btree
+    t.index ["project_id", "ref", "id"], name: "index_ci_pipelines_on_project_idandrefandiddesc", order: { id: :desc }, using: :btree
     t.index ["project_id", "ref", "status", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_status_and_id", using: :btree
     t.index ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree
     t.index ["project_id", "source"], name: "index_ci_pipelines_on_project_id_and_source", using: :btree