diff options
author | Oswaldo Ferreira <oswaldo@gitlab.com> | 2017-12-07 15:41:30 -0200 |
---|---|---|
committer | Oswaldo Ferreira <oswaldo@gitlab.com> | 2018-01-02 17:45:25 -0200 |
commit | bf8c20729baffbf55605484230ff33680d72139b (patch) | |
tree | 70d9bea9ea2a37dc8cd8c37cda396fcf246db72b /db | |
parent | 2cbb2d0eceaed0f31c92d4eed8932e98f4f74559 (diff) | |
download | gitlab-ce-bf8c20729baffbf55605484230ff33680d72139b.tar.gz |
Cache merged and closed events data in merge_request_metrics table
Diffstat (limited to 'db')
3 files changed, 110 insertions, 0 deletions
diff --git a/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb b/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb new file mode 100644 index 00000000000..18af697cf88 --- /dev/null +++ b/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb @@ -0,0 +1,37 @@ +class AddEventsRelatedColumnsToMergeRequestMetrics < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + change_table :merge_request_metrics do |t| + t.references :merged_by, references: :users + t.references :latest_closed_by, references: :users + end + + add_column :merge_request_metrics, :latest_closed_at, :datetime_with_timezone + + add_concurrent_foreign_key :merge_request_metrics, :users, + column: :merged_by_id, + on_delete: :nullify + + add_concurrent_foreign_key :merge_request_metrics, :users, + column: :latest_closed_by_id, + on_delete: :nullify + end + + def down + if foreign_keys_for(:merge_request_metrics, :merged_by_id).any? + remove_foreign_key :merge_request_metrics, column: :merged_by_id + end + + if foreign_keys_for(:merge_request_metrics, :latest_closed_by_id).any? + remove_foreign_key :merge_request_metrics, column: :latest_closed_by_id + end + + remove_columns :merge_request_metrics, + :merged_by_id, :latest_closed_by_id, :latest_closed_at + end +end diff --git a/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb new file mode 100644 index 00000000000..547cc68e10e --- /dev/null +++ b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true +# rubocop:disable GitlabSecurity/SqlInjection + +class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migration + DOWNTIME = false + BATCH_SIZE = 10_000 + MIGRATION = 'PopulateMergeRequestMetricsWithEventsData' + + disable_ddl_transaction! + + class MergeRequest < ActiveRecord::Base + self.table_name = 'merge_requests' + + include ::EachBatch + end + + def up + merge_requests = MergeRequest.where("id IN (#{updatable_merge_requests_union_sql})").reorder(:id) + + say 'Scheduling `PopulateMergeRequestMetricsWithEventsData` jobs' + # It will update around 4_000_000 records in batches of 10_000 merge + # requests (running between 10 minutes) and should take around 66 hours to complete. + # Apparently, production PostgreSQL is able to vacuum 10k-20k dead_tuples by + # minute, and at maximum, each of these jobs should UPDATE 20k records. + # + # More information about the updates in `PopulateMergeRequestMetricsWithEventsData` class. + # + merge_requests.each_batch(of: BATCH_SIZE) do |relation, index| + range = relation.pluck('MIN(id)', 'MAX(id)').first + + BackgroundMigrationWorker.perform_in(index * 10.minutes, MIGRATION, range) + end + end + + def down + execute "update merge_request_metrics set latest_closed_at = null" + execute "update merge_request_metrics set latest_closed_by_id = null" + execute "update merge_request_metrics set merged_by_id = null" + end + + private + + # On staging: + # Planning time: 0.682 ms + # Execution time: 22033.158 ms + # + def updatable_merge_requests_union_sql + metrics_not_exists_clause = + 'NOT EXISTS (SELECT 1 FROM merge_request_metrics WHERE merge_request_metrics.merge_request_id = merge_requests.id)' + + without_metrics_data = <<-SQL.strip_heredoc + merge_request_metrics.merged_by_id IS NULL OR + merge_request_metrics.latest_closed_by_id IS NULL OR + merge_request_metrics.latest_closed_at IS NULL + SQL + + mrs_without_metrics_record = MergeRequest + .where(metrics_not_exists_clause) + .select(:id) + + mrs_without_events_data = MergeRequest + .joins('INNER JOIN merge_request_metrics ON merge_requests.id = merge_request_metrics.merge_request_id') + .where(without_metrics_data) + .select(:id) + + Gitlab::SQL::Union.new([mrs_without_metrics_record, mrs_without_events_data]).to_sql + end +end diff --git a/db/schema.rb b/db/schema.rb index 42715d5e1e8..9d3b5db117e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1056,6 +1056,9 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.integer "pipeline_id" + t.integer "merged_by_id" + t.integer "latest_closed_by_id" + t.datetime_with_timezone "latest_closed_at" end add_index "merge_request_metrics", ["first_deployed_to_production_at"], name: "index_merge_request_metrics_on_first_deployed_to_production_at", using: :btree @@ -1995,6 +1998,8 @@ ActiveRecord::Schema.define(version: 20171220191323) do add_foreign_key "merge_request_diffs", "merge_requests", name: "fk_8483f3258f", on_delete: :cascade add_foreign_key "merge_request_metrics", "ci_pipelines", column: "pipeline_id", on_delete: :cascade add_foreign_key "merge_request_metrics", "merge_requests", on_delete: :cascade + add_foreign_key "merge_request_metrics", "users", column: "latest_closed_by_id", name: "fk_ae440388cc", on_delete: :nullify + add_foreign_key "merge_request_metrics", "users", column: "merged_by_id", name: "fk_7f28d925f3", on_delete: :nullify add_foreign_key "merge_requests", "ci_pipelines", column: "head_pipeline_id", name: "fk_fd82eae0b9", on_delete: :nullify add_foreign_key "merge_requests", "merge_request_diffs", column: "latest_merge_request_diff_id", name: "fk_06067f5644", on_delete: :nullify add_foreign_key "merge_requests", "milestones", name: "fk_6a5165a692", on_delete: :nullify |