Add latest changes from gitlab-org/gitlab@12-4-stable-ee

19 files changed, 473 insertions, 4 deletions

diff --git a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb
new file mode 100644
index 00000000000..33cbe1a62ef
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      class BaseQueryBuilder
+        include Gitlab::CycleAnalytics::MetricsTables
+
+        delegate :subject_class, to: :stage
+
+        # rubocop: disable CodeReuse/ActiveRecord
+
+        def initialize(stage:, params: {})
+          @stage = stage
+          @params = params
+        end
+
+        def build
+          query = subject_class
+          query = filter_by_parent_model(query)
+          query = filter_by_time_range(query)
+          query = stage.start_event.apply_query_customization(query)
+          query = stage.end_event.apply_query_customization(query)
+          query.where(duration_condition)
+        end
+
+        private
+
+        attr_reader :stage, :params
+
+        def duration_condition
+          stage.end_event.timestamp_projection.gteq(stage.start_event.timestamp_projection)
+        end
+
+        def filter_by_parent_model(query)
+          if parent_class.eql?(Project)
+            if subject_class.eql?(Issue)
+              query.where(project_id: stage.parent_id)
+            elsif subject_class.eql?(MergeRequest)
+              query.where(target_project_id: stage.parent_id)
+            else
+              raise ArgumentError, "unknown subject_class: #{subject_class}"
+            end
+          else
+            raise ArgumentError, "unknown parent_class: #{parent_class}"
+          end
+        end
+
+        def filter_by_time_range(query)
+          from = params.fetch(:from, 30.days.ago)
+          to = params[:to]
+
+          query = query.where(subject_table[:created_at].gteq(from))
+          query = query.where(subject_table[:created_at].lteq(to)) if to
+          query
+        end
+
+        def subject_table
+          subject_class.arel_table
+        end
+
+        def parent_class
+          stage.parent.class
+        end
+
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb
new file mode 100644
index 00000000000..0c0f737f2c9
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      # Arguments:
+      #   stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::GroupStage
+      #   params:
+      #     current_user: an instance of User
+      #     from: DateTime
+      #     to: DateTime
+      class DataCollector
+        include Gitlab::Utils::StrongMemoize
+
+        def initialize(stage:, params: {})
+          @stage = stage
+          @params = params
+        end
+
+        def records_fetcher
+          strong_memoize(:records_fetcher) do
+            RecordsFetcher.new(stage: stage, query: query, params: params)
+          end
+        end
+
+        def median
+          strong_memoize(:median) do
+            Median.new(stage: stage, query: query)
+          end
+        end
+
+        private
+
+        attr_reader :stage, :params
+
+        def query
+          BaseQueryBuilder.new(stage: stage, params: params).build
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb
index 286c393005f..8e70236ce75 100644
--- a/lib/gitlab/analytics/cycle_analytics/default_stages.rb
+++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb
@@ -23,6 +23,10 @@ module Gitlab
           ]
         end
 
+        def self.names
+          all.map { |stage| stage[:name] }
+        end
+
         def self.params_for_issue_stage
           {
             name: 'issue',
@@ -88,8 +92,8 @@ module Gitlab
             name: 'production',
             custom: false,
             relative_position: 7,
-            start_event_identifier: :merge_request_merged,
-            end_event_identifier: :merge_request_first_deployed_to_production
+            start_event_identifier: :issue_created,
+            end_event_identifier: :production_stage_end
           }
         end
       end
diff --git a/lib/gitlab/analytics/cycle_analytics/median.rb b/lib/gitlab/analytics/cycle_analytics/median.rb
new file mode 100644
index 00000000000..41883a80338
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/median.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      class Median
+        include StageQueryHelpers
+
+        def initialize(stage:, query:)
+          @stage = stage
+          @query = query
+        end
+
+        def seconds
+          @query = @query.select(median_duration_in_seconds.as('median'))
+          result = execute_query(@query).first || {}
+
+          result['median'] ? result['median'].to_i : nil
+        end
+
+        private
+
+        attr_reader :stage
+
+        def percentile_cont
+          percentile_cont_ordering = Arel::Nodes::UnaryOperation.new(Arel::Nodes::SqlLiteral.new('ORDER BY'), duration)
+          Arel::Nodes::NamedFunction.new(
+            'percentile_cont(0.5) WITHIN GROUP',
+            [percentile_cont_ordering]
+          )
+        end
+
+        def median_duration_in_seconds
+          Arel::Nodes::Extract.new(percentile_cont, :epoch)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
new file mode 100644
index 00000000000..90d03142b2a
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      class RecordsFetcher
+        include Gitlab::Utils::StrongMemoize
+        include StageQueryHelpers
+        include Gitlab::CycleAnalytics::MetricsTables
+
+        MAX_RECORDS = 20
+
+        MAPPINGS = {
+          Issue => {
+            finder_class: IssuesFinder,
+            serializer_class: AnalyticsIssueSerializer,
+            includes_for_query: { project: [:namespace], author: [] },
+            columns_for_select: %I[title iid id created_at author_id project_id]
+          },
+          MergeRequest => {
+            finder_class: MergeRequestsFinder,
+            serializer_class: AnalyticsMergeRequestSerializer,
+            includes_for_query: { target_project: [:namespace], author: [] },
+            columns_for_select: %I[title iid id created_at author_id state target_project_id]
+          }
+        }.freeze
+
+        delegate :subject_class, to: :stage
+
+        def initialize(stage:, query:, params: {})
+          @stage = stage
+          @query = query
+          @params = params
+        end
+
+        def serialized_records
+          strong_memoize(:serialized_records) do
+            # special case (legacy): 'Test' and 'Staging' stages should show Ci::Build records
+            if default_test_stage? || default_staging_stage?
+              AnalyticsBuildSerializer.new.represent(ci_build_records.map { |e| e['build'] })
+            else
+              records.map do |record|
+                project = record.project
+                attributes = record.attributes.merge({
+                  project_path: project.path,
+                  namespace_path: project.namespace.path,
+                  author: record.author
+                })
+                serializer.represent(attributes)
+              end
+            end
+          end
+        end
+
+        private
+
+        attr_reader :stage, :query, :params
+
+        def finder_query
+          MAPPINGS
+            .fetch(subject_class)
+            .fetch(:finder_class)
+            .new(params.fetch(:current_user), finder_params.fetch(stage.parent.class))
+            .execute
+        end
+
+        def columns
+          MAPPINGS.fetch(subject_class).fetch(:columns_for_select).map do |column_name|
+            subject_class.arel_table[column_name]
+          end
+        end
+
+        # EE will override this to include Group rules
+        def finder_params
+          {
+            Project => { project_id: stage.parent_id }
+          }
+        end
+
+        def default_test_stage?
+          stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_test_stage)
+        end
+
+        def default_staging_stage?
+          stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_staging_stage)
+        end
+
+        def serializer
+          MAPPINGS.fetch(subject_class).fetch(:serializer_class).new
+        end
+
+        # Loading Ci::Build records instead of MergeRequest records
+        # rubocop: disable CodeReuse/ActiveRecord
+        def ci_build_records
+          ci_build_join = mr_metrics_table
+            .join(build_table)
+            .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id]))
+            .join_sources
+
+          q = ordered_and_limited_query
+            .joins(ci_build_join)
+            .select(build_table[:id], round_duration_to_seconds.as('total_time'))
+
+          results = execute_query(q).to_a
+
+          Gitlab::CycleAnalytics::Updater.update!(results, from: 'id', to: 'build', klass: ::Ci::Build.includes({ project: [:namespace], user: [], pipeline: [] }))
+        end
+
+        def ordered_and_limited_query
+          query
+            .reorder(stage.end_event.timestamp_projection.desc)
+            .limit(MAX_RECORDS)
+        end
+
+        def records
+          results = finder_query
+            .merge(ordered_and_limited_query)
+            .select(*columns, round_duration_to_seconds.as('total_time'))
+
+          # using preloader instead of includes to avoid AR generating a large column list
+          ActiveRecord::Associations::Preloader.new.preload(
+            results,
+            MAPPINGS.fetch(subject_class).fetch(:includes_for_query)
+          )
+
+          results
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb
index d21f344f483..58572446de6 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb
@@ -18,7 +18,8 @@ module Gitlab
           StageEvents::MergeRequestMerged => 104,
           StageEvents::CodeStageStart => 1_000,
           StageEvents::IssueStageEnd => 1_001,
-          StageEvents::PlanStageStart => 1_002
+          StageEvents::PlanStageStart => 1_002,
+          StageEvents::ProductionStageEnd => 1_003
         }.freeze
 
         EVENTS = ENUM_MAPPING.keys.freeze
@@ -32,7 +33,8 @@ module Gitlab
             StageEvents::MergeRequestCreated
           ],
           StageEvents::IssueCreated => [
-            StageEvents::IssueStageEnd
+            StageEvents::IssueStageEnd,
+            StageEvents::ProductionStageEnd
           ],
           StageEvents::MergeRequestCreated => [
             StageEvents::MergeRequestMerged
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
index ff9c8a79225..6af1b90bccc 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
@@ -16,6 +16,21 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            issue_metrics_table[:first_mentioned_in_commit_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            issue_metrics_join = mr_closing_issues_table
+              .join(issue_metrics_table)
+              .on(mr_closing_issues_table[:issue_id].eq(issue_metrics_table[:issue_id]))
+              .join_sources
+
+            query.joins(:merge_requests_closing_issues).joins(issue_metrics_join)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
index a601c9797f8..8c9a80740a9 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
@@ -16,6 +16,10 @@ module Gitlab
           def object_type
             Issue
           end
+
+          def timestamp_projection
+            issue_table[:created_at]
+          end
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
index 7424043ef7b..fe7f2d85f8b 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             Issue
           end
+
+          def timestamp_projection
+            issue_metrics_table[:first_mentioned_in_commit_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
index ceb229c552f..77e4092b9ab 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
@@ -16,6 +16,19 @@ module Gitlab
           def object_type
             Issue
           end
+
+          def timestamp_projection
+            Arel::Nodes::NamedFunction.new('COALESCE', [
+              issue_metrics_table[:first_associated_with_milestone_at],
+              issue_metrics_table[:first_added_to_board_at]
+            ])
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics).where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil)))
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
index 8be00831b4f..7059c425b8f 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
@@ -16,6 +16,10 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_table[:created_at]
+          end
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
index 6d7a2c023ff..3d7482eaaf0 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_metrics_table[:first_deployed_to_production_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics).where(timestamp_projection.gteq(mr_table[:created_at]))
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
index 12d82fe2c62..36bb4d6fc8d 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_metrics_table[:latest_build_finished_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
index 9e749b0fdfa..468d9899cc7 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_metrics_table[:latest_build_started_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
index bbfb5d12992..82ecaf1cd6b 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_metrics_table[:merged_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
index 803317d8b55..7ece7d62faa 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
@@ -16,6 +16,22 @@ module Gitlab
           def object_type
             Issue
           end
+
+          def timestamp_projection
+            Arel::Nodes::NamedFunction.new('COALESCE', [
+              issue_metrics_table[:first_associated_with_milestone_at],
+              issue_metrics_table[:first_added_to_board_at]
+            ])
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query
+              .joins(:metrics)
+              .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil)))
+              .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil))
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb
new file mode 100644
index 00000000000..607371a32e8
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        class ProductionStageEnd < SimpleStageEvent
+          def self.name
+            PlanStageStart.name
+          end
+
+          def self.identifier
+            :production_stage_end
+          end
+
+          def object_type
+            Issue
+          end
+
+          def timestamp_projection
+            mr_metrics_table[:first_deployed_to_production_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(merge_requests_closing_issues: { merge_request: [:metrics] }).where(mr_metrics_table[:first_deployed_to_production_at].gteq(mr_table[:created_at]))
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
index a55eee048c2..aa392140eb5 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
@@ -6,6 +6,8 @@ module Gitlab
       module StageEvents
         # Base class for expressing an event that can be used for a stage.
         class StageEvent
+          include Gitlab::CycleAnalytics::MetricsTables
+
           def initialize(params)
             @params = params
           end
@@ -21,6 +23,21 @@ module Gitlab
           def object_type
             raise NotImplementedError
           end
+
+          # Each StageEvent must expose a timestamp or a timestamp like expression in order to build a range query.
+          # Example: get me all the Issue records between start event end end event
+          def timestamp_projection
+            raise NotImplementedError
+          end
+
+          # Optionally a StageEvent may apply additional filtering or join other tables on the base query.
+          def apply_query_customization(query)
+            query
+          end
+
+          private
+
+          attr_reader :params
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
new file mode 100644
index 00000000000..34c726b2254
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageQueryHelpers
+        def execute_query(query)
+          ActiveRecord::Base.connection.execute(query.to_sql)
+        end
+
+        def zero_interval
+          Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")])
+        end
+
+        def round_duration_to_seconds
+          Arel::Nodes::Extract.new(duration, :epoch)
+        end
+
+        def duration
+          Arel::Nodes::Subtraction.new(
+            stage.end_event.timestamp_projection,
+            stage.start_event.timestamp_projection
+          )
+        end
+      end
+    end
+  end
+end