Improve performance of the cycle analytics page.

1. These changes bring down page load time for 100 issues from more than
   a minute to about 1.5 seconds.

2. This entire commit is composed of these types of performance
   enhancements:

     - Cache relevant data in `IssueMetrics` wherever possible.
     - Cache relevant data in `MergeRequestMetrics` wherever possible.
     - Preload metrics

3. Given these improvements, we now only need to make 4 SQL calls:

    - Load all issues
    - Load all merge requests
    - Load all metrics for the issues
    - Load all metrics for the merge requests

4. A list of all the data points that are now being pre-calculated:

    a. The first time an issue is mentioned in a commit

      - In `GitPushService`, find all issues mentioned by the given commit
        using `ReferenceExtractor`. Set the `first_mentioned_in_commit_at`
        flag for each of them.

      - There seems to be a (pre-existing) bug here - files (and
        therefore commits) created using the Web CI don't have
        cross-references created, and issues are not closed even when
        the commit title is "Fixes #xx".

    b. The first time a merge request is deployed to production

      When a `Deployment` is created, find all merge requests that
      were merged in before the deployment, and set the
      `first_deployed_to_production_at` flag for each of them.

    c. The start / end time for a merge request pipeline

      Hook into the `Pipeline` state machine. When the `status` moves to
      `running`, find the merge requests whose tip commit matches the
      pipeline, and record the `latest_build_started_at` time for each
      of them. When the `status` moves to `success`, record the
      `latest_build_finished_at` time.

    d. The merge requests that close an issue

      - This was a big cause of the performance problems we were having
        with Cycle Analytics. We need to use `ReferenceExtractor` to make
        this calculation, which is slow when we have to run it on a large
        number of merge requests.

      - When a merge request is created, updated, or refreshed, find the
        issues it closes, and create an instance of
        `MergeRequestsClosingIssues`, which acts as a join model between
        merge requests and issues.

      - If a `MergeRequestsClosingIssues` instance links a merge request
        and an issue, that issue closes that merge request.

5. The `Queries` module was changed into a class, so we can cache the
   results of `issues` and `merge_requests_closing_issues` across
   various cycle analytics stages.

6. The code added in this commit is untested. Tests will be added in the
   next commit.

1 files changed, 55 insertions, 96 deletions

diff --git a/app/models/cycle_analytics/queries.rb b/app/models/cycle_analytics/queries.rb
index 861460f1de4..7bdf1fb6290 100644
--- a/app/models/cycle_analytics/queries.rb
+++ b/app/models/cycle_analytics/queries.rb
@@ -1,121 +1,80 @@
 class CycleAnalytics
-  module Queries
-    class << self
-      def issues(project, created_after:)
-        project.issues.where("created_at >= ?", created_after).preload(:metrics, :system_notes).map { |issue| { issue: issue } }
-      end
-
-      def merge_requests_closing_issues(project, options = {})
-        issues(project, options).map do |data_point|
-          merge_requests = data_point[:issue].closed_by_merge_requests(nil, check_if_open: false)
-          merge_requests.map { |merge_request| { issue: data_point[:issue], merge_request: merge_request } }
-        end.flatten
-      end
+  class Queries
+    def initialize(project)
+      @project = project
+    end
 
-      def issue_first_associated_with_milestone_at
-        lambda do |data_point|
-          issue = data_point[:issue]
-          issue.metrics.first_associated_with_milestone_at if issue.metrics.present?
+    def issues(options = {})
+      @issues_data ||=
+        begin
+          issues_query(options).preload(:metrics).map { |issue| { issue: issue } }
         end
-      end
+    end
 
-      def issue_first_added_to_list_label_at
-        lambda do |data_point|
-          issue = data_point[:issue]
-          issue.metrics.first_added_to_board_at if issue.metrics.present?
-        end
-      end
+    def merge_requests_closing_issues(options = {})
+      @merge_requests_closing_issues_data ||=
+        begin
+          merge_requests_closing_issues = MergeRequestsClosingIssues.where(issue: issues_query(options)).preload(issue: [:metrics], merge_request: [:metrics])
 
-      def issue_first_mentioned_in_commit_at
-        lambda do |data_point|
-          issue = data_point[:issue]
-          commits_mentioning_issue = issue.system_notes.map { |note| note.all_references.commits }.flatten
-          commits_mentioning_issue.map(&:committed_date).min if commits_mentioning_issue.present?
+          merge_requests_closing_issues.map do |record|
+            { issue: record.issue, merge_request: record.merge_request }
+          end
         end
-      end
+    end
 
-      def merge_request_first_closed_at
-        lambda do |data_point|
-          merge_request = data_point[:merge_request]
-          merge_request.metrics.first_closed_at if merge_request.metrics.present?
-        end
+    def issue_first_associated_with_milestone_at
+      lambda do |data_point|
+        issue = data_point[:issue]
+        issue.metrics.first_associated_with_milestone_at if issue.metrics.present?
       end
+    end
 
-      def merge_request_merged_at
-        lambda do |data_point|
-          merge_request = data_point[:merge_request]
-          merge_request.metrics.merged_at if merge_request.metrics.present?
-        end
+    def issue_first_added_to_list_label_at
+      lambda do |data_point|
+        issue = data_point[:issue]
+        issue.metrics.first_added_to_board_at if issue.metrics.present?
       end
+    end
 
-      def merge_request_build_started_at
-        lambda do |data_point|
-          merge_request = data_point[:merge_request]
-          tip = merge_request.commits.first
-          return unless tip
-
-          pipeline = Ci::Pipeline.success.find_by_sha(tip.sha)
-          pipeline.started_at if pipeline
-        end
+    def issue_first_mentioned_in_commit_at
+      lambda do |data_point|
+        issue = data_point[:issue]
+        issue.metrics.first_mentioned_in_commit_at if issue.metrics.present?
       end
+    end
 
-      def merge_request_build_finished_at
-        lambda do |data_point|
-          merge_request = data_point[:merge_request]
-          tip = merge_request.commits.first
-          return unless tip
-
-          pipeline = Ci::Pipeline.success.find_by_sha(tip.sha)
-          pipeline.finished_at if pipeline
-        end
+    def merge_request_merged_at
+      lambda do |data_point|
+        merge_request = data_point[:merge_request]
+        merge_request.metrics.merged_at if merge_request.metrics.present?
       end
+    end
 
-      def merge_request_deployed_to_any_environment_at
-        lambda do |data_point|
-          merge_request = data_point[:merge_request]
-          if merge_request.metrics.present?
-            deployments = Deployment.where(ref: merge_request.target_branch).where("created_at > ?", merge_request.metrics.merged_at)
-            deployment = deployments.order(:created_at).first
-            deployment.created_at if deployment
-          end
-        end
+    def merge_request_build_started_at
+      lambda do |data_point|
+        merge_request = data_point[:merge_request]
+        merge_request.metrics.latest_build_started_at if merge_request.metrics.present?
       end
+    end
 
-      def merge_request_deployed_to_production_at
-        lambda do |data_point|
-          merge_request = data_point[:merge_request]
-          if merge_request.metrics.present?
-            # The first production deploy to the target branch that occurs after the merge request has been merged in.
-            # TODO: Does this need to account for reverts?
-            deployments = Deployment.joins(:environment).where(ref: merge_request.target_branch, "environments.name" => "production").
-                          where("deployments.created_at > ?", merge_request.metrics.merged_at)
-            deployment = deployments.order(:created_at).first
-            deployment.created_at if deployment
-          end
-        end
+    def merge_request_build_finished_at
+      lambda do |data_point|
+        merge_request = data_point[:merge_request]
+        merge_request.metrics.latest_build_finished_at if merge_request.metrics.present?
       end
+    end
 
-      def issue_closing_merge_request_opened_at
-        lambda do |data_point|
-          issue = data_point[:issue]
-          merge_requests = issue.closed_by_merge_requests(nil, check_if_open: false)
-          merge_requests.map(&:created_at).min if merge_requests.present?
-        end
+    def merge_request_deployed_to_production_at
+      lambda do |data_point|
+        merge_request = data_point[:merge_request]
+        merge_request.metrics.first_deployed_to_production_at if merge_request.metrics.present?
       end
+    end
 
-      def merge_request_wip_flag_first_removed_at
-        lambda do |data_point|
-          merge_request = data_point[:merge_request]
-          merge_request.metrics.wip_flag_first_removed_at if merge_request.metrics.present?
-        end
-      end
+    private
 
-      def merge_request_first_assigned_to_user_other_than_author_at
-        lambda do |data_point|
-          merge_request = data_point[:merge_request]
-          merge_request.metrics.first_assigned_to_user_other_than_author if merge_request.metrics.present?
-        end
-      end
+    def issues_query(created_after:)
+      @project.issues.where("created_at >= ?", created_after)
     end
   end
 end