From a39cddf7b78f71006c7b8884432efe41220becb6 Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Mon, 19 Nov 2018 15:33:20 +0100 Subject: Add support for explicit index order ActiveRecord supports `:order` natively, but it is skipped because of our specific patch to support opclasses. Relevant AR code: https://github.com/rails/rails/blob/5-0-stable/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb#L1167 --- config/initializers/postgresql_opclasses_support.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/config/initializers/postgresql_opclasses_support.rb b/config/initializers/postgresql_opclasses_support.rb index 70b530415f5..b066f3788ec 100644 --- a/config/initializers/postgresql_opclasses_support.rb +++ b/config/initializers/postgresql_opclasses_support.rb @@ -130,9 +130,12 @@ module ActiveRecord where = inddef.scan(/WHERE (.+)$/).flatten[0] using = inddef.scan(/USING (.+?) /).flatten[0].to_sym opclasses = Hash[inddef.scan(/\((.+?)\)(?:$| WHERE )/).flatten[0].split(',').map do |column_and_opclass| - column, opclass = column_and_opclass.split(' ').map(&:strip) - [column, opclass] if opclass - end.compact] + column, opclass = column_and_opclass.split(' ').map(&:strip) + end.reject do |column, opclass| + ['desc', 'asc'].include?(opclass&.downcase) + end.map do |column, opclass| + [column, opclass] if opclass + end.compact] index_attrs = [table_name, index_name, unique, column_names, [], orders, where, nil, using, nil, opclasses] @@ -151,6 +154,9 @@ module ActiveRecord def quoted_columns_for_index(column_names, options = {}) column_opclasses = options[:opclasses] || {} column_names.map {|name| "#{quote_column_name(name)} #{column_opclasses[name]}"} + + quoted_columns = Hash[column_names.map { |name| [name.to_sym, "#{quote_column_name(name)} #{column_opclasses[name]}"] }] + add_options_for_index_columns(quoted_columns, options).values end end end -- cgit v1.2.1 From 50f9817afb3d6cc599c499abf1536215eada5600 Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Mon, 19 Nov 2018 15:39:15 +0100 Subject: Add specific indexes for CI This adds specific indexes to speed up the query mentioned in https://gitlab.com/gitlab-org/gitlab-ce/issues/50763 and to avoid statement timeouts. Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/50763. --- changelogs/unreleased/ab-50763-persist-index.yml | 5 +++++ ...19132520_add_indexes_to_ci_builds_and_pipelines.rb | 19 +++++++++++++++++++ db/schema.rb | 2 ++ 3 files changed, 26 insertions(+) create mode 100644 changelogs/unreleased/ab-50763-persist-index.yml create mode 100644 db/migrate/20181119132520_add_indexes_to_ci_builds_and_pipelines.rb diff --git a/changelogs/unreleased/ab-50763-persist-index.yml b/changelogs/unreleased/ab-50763-persist-index.yml new file mode 100644 index 00000000000..0cf11923c5a --- /dev/null +++ b/changelogs/unreleased/ab-50763-persist-index.yml @@ -0,0 +1,5 @@ +--- +title: Add indexes to speed up CI query. +merge_request: 23188 +author: +type: performance diff --git a/db/migrate/20181119132520_add_indexes_to_ci_builds_and_pipelines.rb b/db/migrate/20181119132520_add_indexes_to_ci_builds_and_pipelines.rb new file mode 100644 index 00000000000..2134131b28f --- /dev/null +++ b/db/migrate/20181119132520_add_indexes_to_ci_builds_and_pipelines.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddIndexesToCiBuildsAndPipelines < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_pipelines, [:project_id, :ref, :id], order: { id: :desc } + add_concurrent_index :ci_builds, [:commit_id, :artifacts_expire_at, :id], where: "type = 'Ci::Build' AND (retried = false OR retried IS NULL) AND name IN ('sast', 'dependency_scanning', 'sast:container', 'container_scanning', 'dast')" + end + + def down + remove_concurrent_index :ci_pipelines, [:project_id, :ref, :id], order: { id: :desc } + remove_concurrent_index :ci_builds, [:commit_id, :artifacts_expire_at, :id], where: "type = 'Ci::Build' AND (retried = false OR retried IS NULL) AND name IN ('sast', 'dependency_scanning', 'sast:container', 'container_scanning', 'dast')" + end +end diff --git a/db/schema.rb b/db/schema.rb index 0af185409a9..e6be4b20c86 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -349,6 +349,7 @@ ActiveRecord::Schema.define(version: 20181219145520) do t.string "token_encrypted" t.index ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree t.index ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree + t.index ["commit_id", "artifacts_expire_at", "id"], name: "index_ci_builds_on_commit_id_and_artifacts_expire_at_and_id", where: "(((type)::text = 'Ci::Build'::text) AND ((retried = false) OR (retried IS NULL)) AND ((name)::text = ANY ((ARRAY['sast'::character varying, 'dependency_scanning'::character varying, 'sast:container'::character varying, 'container_scanning'::character varying, 'dast'::character varying])::text[])))", using: :btree t.index ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree t.index ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree t.index ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree @@ -483,6 +484,7 @@ ActiveRecord::Schema.define(version: 20181219145520) do t.index ["merge_request_id"], name: "index_ci_pipelines_on_merge_request_id", where: "(merge_request_id IS NOT NULL)", using: :btree t.index ["pipeline_schedule_id"], name: "index_ci_pipelines_on_pipeline_schedule_id", using: :btree t.index ["project_id", "iid"], name: "index_ci_pipelines_on_project_id_and_iid", unique: true, where: "(iid IS NOT NULL)", using: :btree + t.index ["project_id", "ref", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_id", order: { id: :desc }, using: :btree t.index ["project_id", "ref", "status", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_status_and_id", using: :btree t.index ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree t.index ["project_id", "source"], name: "index_ci_pipelines_on_project_id_and_source", using: :btree -- cgit v1.2.1 From 00b5026a9fe2e38f6baaef13a2d2450112cffca3 Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Mon, 19 Nov 2018 17:09:20 +0100 Subject: Add specific index names This is useful to distinguish any other indexes on the same columns but with different conditions and/or sorting. --- ...32520_add_indexes_to_ci_builds_and_pipelines.rb | 33 +++++++++++++++++++--- db/schema.rb | 4 +-- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/db/migrate/20181119132520_add_indexes_to_ci_builds_and_pipelines.rb b/db/migrate/20181119132520_add_indexes_to_ci_builds_and_pipelines.rb index 2134131b28f..adbc3928b26 100644 --- a/db/migrate/20181119132520_add_indexes_to_ci_builds_and_pipelines.rb +++ b/db/migrate/20181119132520_add_indexes_to_ci_builds_and_pipelines.rb @@ -8,12 +8,37 @@ class AddIndexesToCiBuildsAndPipelines < ActiveRecord::Migration disable_ddl_transaction! def up - add_concurrent_index :ci_pipelines, [:project_id, :ref, :id], order: { id: :desc } - add_concurrent_index :ci_builds, [:commit_id, :artifacts_expire_at, :id], where: "type = 'Ci::Build' AND (retried = false OR retried IS NULL) AND name IN ('sast', 'dependency_scanning', 'sast:container', 'container_scanning', 'dast')" + indexes.each do |index| + add_concurrent_index(*index) + end end def down - remove_concurrent_index :ci_pipelines, [:project_id, :ref, :id], order: { id: :desc } - remove_concurrent_index :ci_builds, [:commit_id, :artifacts_expire_at, :id], where: "type = 'Ci::Build' AND (retried = false OR retried IS NULL) AND name IN ('sast', 'dependency_scanning', 'sast:container', 'container_scanning', 'dast')" + indexes.each do |index| + remove_concurrent_index(*index) + end + end + + private + + def indexes + [ + [ + :ci_pipelines, + [:project_id, :ref, :id], + { + order: { id: :desc }, + name: 'index_ci_pipelines_on_project_idandrefandiddesc' + } + ], + [ + :ci_builds, + [:commit_id, :artifacts_expire_at, :id], + { + where: "type::text = 'Ci::Build'::text AND (retried = false OR retried IS NULL) AND (name::text = ANY (ARRAY['sast'::character varying, 'dependency_scanning'::character varying, 'sast:container'::character varying, 'container_scanning'::character varying, 'dast'::character varying]::text[]))", + name: 'index_ci_builds_on_commit_id_and_artifacts_expireatandidpartial' + } + ] + ] end end diff --git a/db/schema.rb b/db/schema.rb index e6be4b20c86..dbc0242172c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -349,7 +349,7 @@ ActiveRecord::Schema.define(version: 20181219145520) do t.string "token_encrypted" t.index ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree t.index ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree - t.index ["commit_id", "artifacts_expire_at", "id"], name: "index_ci_builds_on_commit_id_and_artifacts_expire_at_and_id", where: "(((type)::text = 'Ci::Build'::text) AND ((retried = false) OR (retried IS NULL)) AND ((name)::text = ANY ((ARRAY['sast'::character varying, 'dependency_scanning'::character varying, 'sast:container'::character varying, 'container_scanning'::character varying, 'dast'::character varying])::text[])))", using: :btree + t.index ["commit_id", "artifacts_expire_at", "id"], name: "index_ci_builds_on_commit_id_and_artifacts_expireatandidpartial", where: "(((type)::text = 'Ci::Build'::text) AND ((retried = false) OR (retried IS NULL)) AND ((name)::text = ANY (ARRAY[('sast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('sast:container'::character varying)::text, ('container_scanning'::character varying)::text, ('dast'::character varying)::text])))", using: :btree t.index ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree t.index ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree t.index ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree @@ -484,7 +484,7 @@ ActiveRecord::Schema.define(version: 20181219145520) do t.index ["merge_request_id"], name: "index_ci_pipelines_on_merge_request_id", where: "(merge_request_id IS NOT NULL)", using: :btree t.index ["pipeline_schedule_id"], name: "index_ci_pipelines_on_pipeline_schedule_id", using: :btree t.index ["project_id", "iid"], name: "index_ci_pipelines_on_project_id_and_iid", unique: true, where: "(iid IS NOT NULL)", using: :btree - t.index ["project_id", "ref", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_id", order: { id: :desc }, using: :btree + t.index ["project_id", "ref", "id"], name: "index_ci_pipelines_on_project_idandrefandiddesc", order: { id: :desc }, using: :btree t.index ["project_id", "ref", "status", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_status_and_id", using: :btree t.index ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree t.index ["project_id", "source"], name: "index_ci_pipelines_on_project_id_and_source", using: :btree -- cgit v1.2.1