summaryrefslogtreecommitdiff
path: root/app/models
diff options
context:
space:
mode:
authorTimothy Andrew <mail@timothyandrew.net>2016-09-15 14:29:36 +0530
committerTimothy Andrew <mail@timothyandrew.net>2016-09-15 14:53:02 +0530
commitba25e2f1ac61b47940f939a2d9f1d0ad417e1de2 (patch)
treee30ce100687fde731bb738c95a8ccd12033f9c42 /app/models
parent798b17a35311d60fe18440bfc53dba3aadd7b099 (diff)
downloadgitlab-ce-ba25e2f1ac61b47940f939a2d9f1d0ad417e1de2.tar.gz
Improve performance of the cycle analytics page.
1. These changes bring down page load time for 100 issues from more than a minute to about 1.5 seconds. 2. This entire commit is composed of these types of performance enhancements: - Cache relevant data in `IssueMetrics` wherever possible. - Cache relevant data in `MergeRequestMetrics` wherever possible. - Preload metrics 3. Given these improvements, we now only need to make 4 SQL calls: - Load all issues - Load all merge requests - Load all metrics for the issues - Load all metrics for the merge requests 4. A list of all the data points that are now being pre-calculated: a. The first time an issue is mentioned in a commit - In `GitPushService`, find all issues mentioned by the given commit using `ReferenceExtractor`. Set the `first_mentioned_in_commit_at` flag for each of them. - There seems to be a (pre-existing) bug here - files (and therefore commits) created using the Web CI don't have cross-references created, and issues are not closed even when the commit title is "Fixes #xx". b. The first time a merge request is deployed to production When a `Deployment` is created, find all merge requests that were merged in before the deployment, and set the `first_deployed_to_production_at` flag for each of them. c. The start / end time for a merge request pipeline Hook into the `Pipeline` state machine. When the `status` moves to `running`, find the merge requests whose tip commit matches the pipeline, and record the `latest_build_started_at` time for each of them. When the `status` moves to `success`, record the `latest_build_finished_at` time. d. The merge requests that close an issue - This was a big cause of the performance problems we were having with Cycle Analytics. We need to use `ReferenceExtractor` to make this calculation, which is slow when we have to run it on a large number of merge requests. - When a merge request is created, updated, or refreshed, find the issues it closes, and create an instance of `MergeRequestsClosingIssues`, which acts as a join model between merge requests and issues. - If a `MergeRequestsClosingIssues` instance links a merge request and an issue, that issue closes that merge request. 5. The `Queries` module was changed into a class, so we can cache the results of `issues` and `merge_requests_closing_issues` across various cycle analytics stages. 6. The code added in this commit is untested. Tests will be added in the next commit.
Diffstat (limited to 'app/models')
-rw-r--r--app/models/ci/pipeline.rb19
-rw-r--r--app/models/cycle_analytics.rb36
-rw-r--r--app/models/cycle_analytics/queries.rb151
-rw-r--r--app/models/issue.rb3
-rw-r--r--app/models/issue/metrics.rb4
-rw-r--r--app/models/merge_request.rb15
-rw-r--r--app/models/merge_request/metrics.rb12
-rw-r--r--app/models/merge_requests_closing_issues.rb4
8 files changed, 130 insertions, 114 deletions
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index bd1737c7587..1f7e6b11bb9 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -55,6 +55,18 @@ module Ci
pipeline.finished_at = Time.now
end
+ after_transition [:created, :pending] => :running do |pipeline|
+ pipeline.merge_requests.each do |merge_request|
+ merge_request.metrics.record_latest_build_start_time!(pipeline.started_at) if merge_request.metrics.present?
+ end
+ end
+
+ after_transition any => [:success] do |pipeline|
+ pipeline.merge_requests.each do |merge_request|
+ merge_request.metrics.record_latest_build_finish_time!(pipeline.finished_at) if merge_request.metrics.present?
+ end
+ end
+
before_transition do |pipeline|
pipeline.update_duration
end
@@ -267,6 +279,13 @@ module Ci
project.execute_services(data, :pipeline_hooks)
end
+ # Merge requests for which the current pipeline is running against
+ # the merge request's latest commit.
+ def merge_requests
+ project.merge_requests.where(source_branch: self.ref).
+ select { |merge_request| merge_request.pipeline.id == self.id }
+ end
+
private
def pipeline_data
diff --git a/app/models/cycle_analytics.rb b/app/models/cycle_analytics.rb
index 9444f1f31a8..17a80115fe5 100644
--- a/app/models/cycle_analytics.rb
+++ b/app/models/cycle_analytics.rb
@@ -2,8 +2,8 @@ class CycleAnalytics
attr_reader :from
def initialize(project, from:)
- @project = project
@from = from
+ @queries = Queries.new(project)
end
def as_json(options = {})
@@ -14,45 +14,45 @@ class CycleAnalytics
end
def issue
- calculate_metric(Queries::issues(@project, created_after: @from),
+ calculate_metric(@queries.issues(created_after: @from),
-> (data_point) { data_point[:issue].created_at },
- [Queries::issue_first_associated_with_milestone_at, Queries::issue_first_added_to_list_label_at])
+ [@queries.issue_first_associated_with_milestone_at, @queries.issue_first_added_to_list_label_at])
end
def plan
- calculate_metric(Queries::issues(@project, created_after: @from),
- [Queries::issue_first_associated_with_milestone_at, Queries::issue_first_added_to_list_label_at],
- Queries::issue_first_mentioned_in_commit_at)
+ calculate_metric(@queries.issues(created_after: @from),
+ [@queries.issue_first_associated_with_milestone_at, @queries.issue_first_added_to_list_label_at],
+ @queries.issue_first_mentioned_in_commit_at)
end
def code
- calculate_metric(Queries::merge_requests_closing_issues(@project, created_after: @from),
- Queries::issue_first_mentioned_in_commit_at,
+ calculate_metric(@queries.merge_requests_closing_issues(created_after: @from),
+ @queries.issue_first_mentioned_in_commit_at,
-> (data_point) { data_point[:merge_request].created_at })
end
def test
- calculate_metric(Queries::merge_requests_closing_issues(@project, created_after: @from),
- Queries::merge_request_build_started_at,
- Queries::merge_request_build_finished_at)
+ calculate_metric(@queries.merge_requests_closing_issues(created_after: @from),
+ @queries.merge_request_build_started_at,
+ @queries.merge_request_build_finished_at)
end
def review
- calculate_metric(Queries::merge_requests_closing_issues(@project, created_after: @from),
+ calculate_metric(@queries.merge_requests_closing_issues(created_after: @from),
-> (data_point) { data_point[:merge_request].created_at },
- Queries::merge_request_merged_at)
+ @queries.merge_request_merged_at)
end
def staging
- calculate_metric(Queries::merge_requests_closing_issues(@project, created_after: @from),
- Queries::merge_request_merged_at,
- Queries::merge_request_deployed_to_production_at)
+ calculate_metric(@queries.merge_requests_closing_issues(created_after: @from),
+ @queries.merge_request_merged_at,
+ @queries.merge_request_deployed_to_production_at)
end
def production
- calculate_metric(Queries::merge_requests_closing_issues(@project, created_after: @from),
+ calculate_metric(@queries.merge_requests_closing_issues(created_after: @from),
-> (data_point) { data_point[:issue].created_at },
- Queries::merge_request_deployed_to_production_at)
+ @queries.merge_request_deployed_to_production_at)
end
private
diff --git a/app/models/cycle_analytics/queries.rb b/app/models/cycle_analytics/queries.rb
index 861460f1de4..7bdf1fb6290 100644
--- a/app/models/cycle_analytics/queries.rb
+++ b/app/models/cycle_analytics/queries.rb
@@ -1,121 +1,80 @@
class CycleAnalytics
- module Queries
- class << self
- def issues(project, created_after:)
- project.issues.where("created_at >= ?", created_after).preload(:metrics, :system_notes).map { |issue| { issue: issue } }
- end
-
- def merge_requests_closing_issues(project, options = {})
- issues(project, options).map do |data_point|
- merge_requests = data_point[:issue].closed_by_merge_requests(nil, check_if_open: false)
- merge_requests.map { |merge_request| { issue: data_point[:issue], merge_request: merge_request } }
- end.flatten
- end
+ class Queries
+ def initialize(project)
+ @project = project
+ end
- def issue_first_associated_with_milestone_at
- lambda do |data_point|
- issue = data_point[:issue]
- issue.metrics.first_associated_with_milestone_at if issue.metrics.present?
+ def issues(options = {})
+ @issues_data ||=
+ begin
+ issues_query(options).preload(:metrics).map { |issue| { issue: issue } }
end
- end
+ end
- def issue_first_added_to_list_label_at
- lambda do |data_point|
- issue = data_point[:issue]
- issue.metrics.first_added_to_board_at if issue.metrics.present?
- end
- end
+ def merge_requests_closing_issues(options = {})
+ @merge_requests_closing_issues_data ||=
+ begin
+ merge_requests_closing_issues = MergeRequestsClosingIssues.where(issue: issues_query(options)).preload(issue: [:metrics], merge_request: [:metrics])
- def issue_first_mentioned_in_commit_at
- lambda do |data_point|
- issue = data_point[:issue]
- commits_mentioning_issue = issue.system_notes.map { |note| note.all_references.commits }.flatten
- commits_mentioning_issue.map(&:committed_date).min if commits_mentioning_issue.present?
+ merge_requests_closing_issues.map do |record|
+ { issue: record.issue, merge_request: record.merge_request }
+ end
end
- end
+ end
- def merge_request_first_closed_at
- lambda do |data_point|
- merge_request = data_point[:merge_request]
- merge_request.metrics.first_closed_at if merge_request.metrics.present?
- end
+ def issue_first_associated_with_milestone_at
+ lambda do |data_point|
+ issue = data_point[:issue]
+ issue.metrics.first_associated_with_milestone_at if issue.metrics.present?
end
+ end
- def merge_request_merged_at
- lambda do |data_point|
- merge_request = data_point[:merge_request]
- merge_request.metrics.merged_at if merge_request.metrics.present?
- end
+ def issue_first_added_to_list_label_at
+ lambda do |data_point|
+ issue = data_point[:issue]
+ issue.metrics.first_added_to_board_at if issue.metrics.present?
end
+ end
- def merge_request_build_started_at
- lambda do |data_point|
- merge_request = data_point[:merge_request]
- tip = merge_request.commits.first
- return unless tip
-
- pipeline = Ci::Pipeline.success.find_by_sha(tip.sha)
- pipeline.started_at if pipeline
- end
+ def issue_first_mentioned_in_commit_at
+ lambda do |data_point|
+ issue = data_point[:issue]
+ issue.metrics.first_mentioned_in_commit_at if issue.metrics.present?
end
+ end
- def merge_request_build_finished_at
- lambda do |data_point|
- merge_request = data_point[:merge_request]
- tip = merge_request.commits.first
- return unless tip
-
- pipeline = Ci::Pipeline.success.find_by_sha(tip.sha)
- pipeline.finished_at if pipeline
- end
+ def merge_request_merged_at
+ lambda do |data_point|
+ merge_request = data_point[:merge_request]
+ merge_request.metrics.merged_at if merge_request.metrics.present?
end
+ end
- def merge_request_deployed_to_any_environment_at
- lambda do |data_point|
- merge_request = data_point[:merge_request]
- if merge_request.metrics.present?
- deployments = Deployment.where(ref: merge_request.target_branch).where("created_at > ?", merge_request.metrics.merged_at)
- deployment = deployments.order(:created_at).first
- deployment.created_at if deployment
- end
- end
+ def merge_request_build_started_at
+ lambda do |data_point|
+ merge_request = data_point[:merge_request]
+ merge_request.metrics.latest_build_started_at if merge_request.metrics.present?
end
+ end
- def merge_request_deployed_to_production_at
- lambda do |data_point|
- merge_request = data_point[:merge_request]
- if merge_request.metrics.present?
- # The first production deploy to the target branch that occurs after the merge request has been merged in.
- # TODO: Does this need to account for reverts?
- deployments = Deployment.joins(:environment).where(ref: merge_request.target_branch, "environments.name" => "production").
- where("deployments.created_at > ?", merge_request.metrics.merged_at)
- deployment = deployments.order(:created_at).first
- deployment.created_at if deployment
- end
- end
+ def merge_request_build_finished_at
+ lambda do |data_point|
+ merge_request = data_point[:merge_request]
+ merge_request.metrics.latest_build_finished_at if merge_request.metrics.present?
end
+ end
- def issue_closing_merge_request_opened_at
- lambda do |data_point|
- issue = data_point[:issue]
- merge_requests = issue.closed_by_merge_requests(nil, check_if_open: false)
- merge_requests.map(&:created_at).min if merge_requests.present?
- end
+ def merge_request_deployed_to_production_at
+ lambda do |data_point|
+ merge_request = data_point[:merge_request]
+ merge_request.metrics.first_deployed_to_production_at if merge_request.metrics.present?
end
+ end
- def merge_request_wip_flag_first_removed_at
- lambda do |data_point|
- merge_request = data_point[:merge_request]
- merge_request.metrics.wip_flag_first_removed_at if merge_request.metrics.present?
- end
- end
+ private
- def merge_request_first_assigned_to_user_other_than_author_at
- lambda do |data_point|
- merge_request = data_point[:merge_request]
- merge_request.metrics.first_assigned_to_user_other_than_author if merge_request.metrics.present?
- end
- end
+ def issues_query(created_after:)
+ @project.issues.where("created_at >= ?", created_after)
end
end
end
diff --git a/app/models/issue.rb b/app/models/issue.rb
index cec5e456c94..fd66a7a2714 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -25,6 +25,9 @@ class Issue < ActiveRecord::Base
has_one :metrics, dependent: :destroy
+ has_many :merge_requests_closing_issues, class_name: MergeRequestsClosingIssues
+ has_many :closed_by_merge_requests, through: :merge_requests_closing_issues, source: :merge_request
+
validates :project, presence: true
scope :cared, ->(user) { where(assignee_id: user) }
diff --git a/app/models/issue/metrics.rb b/app/models/issue/metrics.rb
index 4436696cc1a..3c1fcea9e6f 100644
--- a/app/models/issue/metrics.rb
+++ b/app/models/issue/metrics.rb
@@ -13,6 +13,10 @@ class Issue::Metrics < ActiveRecord::Base
self.save if self.changed?
end
+ def record_commit_mention!(commit_time)
+ self.update(first_mentioned_in_commit_at: commit_time) if self.first_mentioned_in_commit_at.blank?
+ end
+
private
def issue_assigned_to_list_label?
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 8127c2cdd8d..14e093eed93 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -17,6 +17,9 @@ class MergeRequest < ActiveRecord::Base
has_many :events, as: :target, dependent: :destroy
+ has_many :merge_requests_closing_issues, class_name: MergeRequestsClosingIssues
+ has_many :issues_closed, through: :merge_requests_closing_issues, source: :issue
+
serialize :merge_params, Hash
after_create :ensure_merge_request_diff, unless: :importing?
@@ -494,6 +497,18 @@ class MergeRequest < ActiveRecord::Base
target_project
end
+ # If the merge request closes any issues, save this information in the
+ # `MergeRequestsClosingIssues` model. This is a performance optimization.
+ # Calculating this information for a number of merge requests requires
+ # running `ReferenceExtractor` on each of them separately.
+ def cache_merge_request_closes_issues!(current_user = self.author)
+ transaction do
+ closes_issues(current_user).each do |issue|
+ MergeRequestsClosingIssues.create!(merge_request: self, issue: issue)
+ end
+ end
+ end
+
def closes_issue?(issue)
closes_issues.include?(issue)
end
diff --git a/app/models/merge_request/metrics.rb b/app/models/merge_request/metrics.rb
index a0bc41c9cc1..18b1e7c5bf3 100644
--- a/app/models/merge_request/metrics.rb
+++ b/app/models/merge_request/metrics.rb
@@ -20,4 +20,16 @@ class MergeRequest::Metrics < ActiveRecord::Base
self.save if self.changed?
end
+
+ def record_production_deploy!(deploy_time)
+ self.update(first_deployed_to_production_at: deploy_time) if self.first_deployed_to_production_at.blank?
+ end
+
+ def record_latest_build_start_time!(start_time)
+ self.update(latest_build_started_at: start_time, latest_build_finished_at: nil)
+ end
+
+ def record_latest_build_finish_time!(finish_time)
+ self.update(latest_build_finished_at: finish_time)
+ end
end
diff --git a/app/models/merge_requests_closing_issues.rb b/app/models/merge_requests_closing_issues.rb
new file mode 100644
index 00000000000..4f093fa27c5
--- /dev/null
+++ b/app/models/merge_requests_closing_issues.rb
@@ -0,0 +1,4 @@
+class MergeRequestsClosingIssues < ActiveRecord::Base
+ belongs_to :merge_request
+ belongs_to :issue
+end