New Cycle Analytics backendnew-cycle-analytics-stage-backend

This change lays the foundation for customizable cycle analytics stages.
The main reason for the change is to extract the event definitions to
separate objects (start_event, end_event) so that it could be easily
customized later on.

In this commit the default stages that we provide in CE are migrated to
the new structure.

52 files changed, 1521 insertions, 19 deletions

diff --git a/app/controllers/projects/cycle_analytics/records_controller.rb b/app/controllers/projects/cycle_analytics/records_controller.rb
new file mode 100644
index 00000000000..00433a453f7
--- /dev/null
+++ b/app/controllers/projects/cycle_analytics/records_controller.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Projects
+  module CycleAnalytics
+    class RecordsController < Projects::ApplicationController
+      include CycleAnalyticsParams
+
+      before_action :authorize_read_cycle_analytics!
+
+      def index
+        stage = ::CycleAnalytics::StageFindService.new(parent: project, id: params[:stage_id]).execute
+        data_collector = Gitlab::CycleAnalytics::DataCollector.new(stage, {
+          from: cycle_analytics_params[:start_date],
+          current_user: current_user
+        })
+
+        render json: data_collector.records_fetcher.serialized_records
+      end
+
+      def cycle_analytics_params
+        return {} unless params[:cycle_analytics].present?
+
+        params[:cycle_analytics].permit(:start_date)
+      end
+    end
+  end
+end
diff --git a/app/controllers/projects/cycle_analytics_controller.rb b/app/controllers/projects/cycle_analytics_controller.rb
index 2d46a71bf99..40948fd939a 100644
--- a/app/controllers/projects/cycle_analytics_controller.rb
+++ b/app/controllers/projects/cycle_analytics_controller.rb
@@ -11,22 +11,39 @@ class Projects::CycleAnalyticsController < Projects::ApplicationController
   def show
     @cycle_analytics = ::CycleAnalytics::ProjectLevel.new(@project, options: options(cycle_analytics_params))
 
-    @cycle_analytics_no_data = @cycle_analytics.no_stats?
-
     respond_to do |format|
-      format.html
-      format.json { render json: cycle_analytics_json }
+      format.html do
+        @cycle_analytics_no_data = @cycle_analytics.no_stats?
+
+        render :show
+      end
+      format.json do
+        if params[:new_api] # Use new API when FE is ready
+          stages = ::CycleAnalytics::StageListService.new(parent: project).execute
+          render json: ::CycleAnalytics::CycleAnalyticsEntity.new({
+            stages: stages,
+            summary: @cycle_analytics.summary,
+            permissions: @cycle_analytics.permissions(user: current_user)
+          })
+        else
+          render json: cycle_analytics_json
+        end
+      end
     end
   end
 
-  private
-
-  def cycle_analytics_params
-    return {} unless params[:cycle_analytics].present?
+  def median
+    stage = ::CycleAnalytics::StageFindService.new(parent: project, id: params[:stage_id]).execute
+    data_collector = Gitlab::CycleAnalytics::DataCollector.new(stage, {
+      from: cycle_analytics_params[:start_date],
+      current_user: current_user
+    })
 
-    params[:cycle_analytics].permit(:start_date)
+    render json: ::CycleAnalytics::MedianEntity.new(data_collector.median.seconds)
   end
 
+  private
+
   def cycle_analytics_json
     {
       summary: @cycle_analytics.summary,
@@ -35,6 +52,12 @@ class Projects::CycleAnalyticsController < Projects::ApplicationController
     }
   end
 
+  def cycle_analytics_params
+    return {} unless params[:cycle_analytics].present?
+
+    params[:cycle_analytics].permit(:start_date)
+  end
+
   def whitelist_query_limiting
     Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42671')
   end
diff --git a/app/models/concerns/cycle_analytics/stage.rb b/app/models/concerns/cycle_analytics/stage.rb
new file mode 100644
index 00000000000..63a0db451b1
--- /dev/null
+++ b/app/models/concerns/cycle_analytics/stage.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module CycleAnalytics
+  module Stage
+    extend ActiveSupport::Concern
+
+    included do
+      validates :name, presence: true
+      validate :validate_stage_event_pairs
+
+      enum start_event_identifier: Gitlab::CycleAnalytics::StageEvents.to_enum, _prefix: :start_event_identifier
+      enum end_event_identifier: Gitlab::CycleAnalytics::StageEvents.to_enum, _prefix: :end_event_identifier
+
+      alias_attribute :custom_stage?, :custom
+    end
+
+    def parent=(_)
+      raise NotImplementedError
+    end
+
+    def parent
+      raise NotImplementedError
+    end
+
+    def start_event
+      Gitlab::CycleAnalytics::StageEvents[start_event_identifier].new(params_for_start_event)
+    end
+
+    def end_event
+      Gitlab::CycleAnalytics::StageEvents[end_event_identifier].new(params_for_end_event)
+    end
+
+    def params_for_start_event
+      {}
+    end
+
+    def params_for_end_event
+      {}
+    end
+
+    def default_stage?
+      !custom
+    end
+
+    # The model that is going to be queried, Issue or MergeRequest
+    def subject_model
+      start_event.object_type
+    end
+
+    def matches_with_stage_params?(stage_params)
+      default_stage? &&
+        start_event_identifier == stage_params[:start_event_identifier] &&
+        end_event_identifier == stage_params[:end_event_identifier]
+    end
+
+    private
+
+    def validate_stage_event_pairs
+      return if start_event_identifier.nil? || end_event_identifier.nil?
+
+      unless Gitlab::CycleAnalytics::StageEvents.pairing_rules.fetch(start_event.class, []).include?(end_event.class)
+        errors.add(:end_event, :not_allowed_for_the_given_start_event)
+      end
+    end
+  end
+end
diff --git a/app/models/cycle_analytics.rb b/app/models/cycle_analytics.rb
new file mode 100644
index 00000000000..f64010e89f4
--- /dev/null
+++ b/app/models/cycle_analytics.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module CycleAnalytics
+  def self.table_name_prefix
+    'cycle_analytics_'
+  end
+end
diff --git a/app/models/cycle_analytics/project_stage.rb b/app/models/cycle_analytics/project_stage.rb
new file mode 100644
index 00000000000..8f0edcf0ed6
--- /dev/null
+++ b/app/models/cycle_analytics/project_stage.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module CycleAnalytics
+  class ProjectStage < ApplicationRecord
+    include CycleAnalytics::Stage
+
+    belongs_to :project
+
+    alias_attribute :parent, :project
+  end
+end
diff --git a/app/models/project.rb b/app/models/project.rb
index 960795b73cb..4b8ca831ca7 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -284,6 +284,7 @@ class Project < ApplicationRecord
   has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
   has_many :remote_mirrors, inverse_of: :project
+  has_many :cycle_analytics_stages, class_name: 'CycleAnalytics::ProjectStage'
 
   accepts_nested_attributes_for :variables, allow_destroy: true
   accepts_nested_attributes_for :project_feature, update_only: true
diff --git a/app/serializers/cycle_analytics/cycle_analytics_entity.rb b/app/serializers/cycle_analytics/cycle_analytics_entity.rb
new file mode 100644
index 00000000000..423ba63696d
--- /dev/null
+++ b/app/serializers/cycle_analytics/cycle_analytics_entity.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class CycleAnalytics::CycleAnalyticsEntity < Grape::Entity
+  include RequestAwareEntity
+
+  expose :events, using: CycleAnalytics::EventEntity
+  expose :stages, using: CycleAnalytics::StageEntity
+  expose :summary
+  expose :permissions
+
+  def events
+    Gitlab::CycleAnalytics::StageEvents.events
+  end
+end
diff --git a/app/serializers/cycle_analytics/event_entity.rb b/app/serializers/cycle_analytics/event_entity.rb
new file mode 100644
index 00000000000..98de5d6768a
--- /dev/null
+++ b/app/serializers/cycle_analytics/event_entity.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class CycleAnalytics::EventEntity < Grape::Entity
+  expose :name
+  expose :identifier
+  expose :type
+  expose :can_be_start_event
+  expose :allowed_end_events
+
+  private
+
+  def type
+    'simple'
+  end
+
+  def can_be_start_event
+    Gitlab::CycleAnalytics::StageEvents.pairing_rules.has_key?(object)
+  end
+
+  def allowed_end_events
+    Gitlab::CycleAnalytics::StageEvents.pairing_rules.fetch(object, []).map(&:identifier)
+  end
+end
diff --git a/app/serializers/cycle_analytics/median_entity.rb b/app/serializers/cycle_analytics/median_entity.rb
new file mode 100644
index 00000000000..08daa919115
--- /dev/null
+++ b/app/serializers/cycle_analytics/median_entity.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class CycleAnalytics::MedianEntity < Grape::Entity
+  include EntityDateHelper
+
+  expose :value
+
+  private
+
+  def value
+    object.nil? ? nil : distance_of_time_in_words(object)
+  end
+end
diff --git a/app/serializers/cycle_analytics/stage_decorator.rb b/app/serializers/cycle_analytics/stage_decorator.rb
new file mode 100644
index 00000000000..3f170cf7e59
--- /dev/null
+++ b/app/serializers/cycle_analytics/stage_decorator.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+class CycleAnalytics::StageDecorator < SimpleDelegator
+  DEFAULT_STAGE_ATTRIBUTES = {
+    issue: {
+      title: -> { s_('CycleAnalyticsStage|Issue') },
+      description: -> { _("Time before an issue gets scheduled") }
+    },
+    plan: {
+      title: -> { s_('CycleAnalyticsStage|Plan') },
+      description: -> { _("Time before an issue starts implementation") }
+    },
+    code: {
+      title: -> { s_('CycleAnalyticsStage|Code') },
+      description: -> { _("Time until first merge request") }
+    },
+    test: {
+      title: -> { s_('CycleAnalyticsStage|Test') },
+      description: -> { _("Total test time for all commits/merges") }
+    },
+    review: {
+      title: -> { s_('CycleAnalyticsStage|Review') },
+      description: -> { _("Time between merge request creation and merge/close") }
+    },
+    staging: {
+      title: -> { s_('CycleAnalyticsStage|Staging') },
+      description: -> { _("From merge request merge until deploy to production") }
+    },
+    production: {
+      title: -> { s_('CycleAnalyticsStage|Review') },
+      description: -> { _("From issue creation until deploy to production") }
+    }
+  }.freeze
+
+  def title
+    extract_default_stage_attribute(:title) || name
+  end
+
+  def description
+    extract_default_stage_attribute(:description) || ''
+  end
+
+  def legend
+    if matches_with_stage_params?(Gitlab::CycleAnalytics::DefaultStages.params_for_test_stage)
+      _("Related Jobs")
+    elsif matches_with_stage_params?(Gitlab::CycleAnalytics::DefaultStages.params_for_staging_stage)
+      _("Related Deployed Jobs")
+    elsif subject_model.eql?(Issue)
+      _("Related Issues")
+    elsif subject_model.eql?(MergeRequest)
+      _("Related Merged Requests")
+    end
+  end
+
+  private
+
+  def extract_default_stage_attribute(attribute)
+    DEFAULT_STAGE_ATTRIBUTES.dig(name.to_sym, attribute.to_sym)&.call
+  end
+end
diff --git a/app/serializers/cycle_analytics/stage_entity.rb b/app/serializers/cycle_analytics/stage_entity.rb
new file mode 100644
index 00000000000..39fc87dc05e
--- /dev/null
+++ b/app/serializers/cycle_analytics/stage_entity.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class CycleAnalytics::StageEntity < Grape::Entity
+  expose :name
+  expose :legend
+  expose :description
+  expose :id
+  expose :relative_position, as: :position
+  expose :hidden
+  expose :custom
+  expose :start_event_identifier
+  expose :end_event_identifier
+
+  def initialize(object, options = {})
+    super(CycleAnalytics::StageDecorator.new(object), options)
+  end
+
+  def id
+    object.id || object.name
+  end
+end
diff --git a/app/services/cycle_analytics/stage_find_service.rb b/app/services/cycle_analytics/stage_find_service.rb
new file mode 100644
index 00000000000..12e343d71be
--- /dev/null
+++ b/app/services/cycle_analytics/stage_find_service.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module CycleAnalytics
+  class StageFindService
+    def initialize(parent:, id:)
+      @parent = parent
+      @id = id
+    end
+
+    def execute
+      find_in_memory_stage_by_name!
+    end
+
+    private
+
+    attr_reader :parent, :id
+
+    def find_in_memory_stage_by_name!
+      raw_stage = Gitlab::CycleAnalytics::DefaultStages.all.find do |hash|
+        hash[:name].eql?(id.to_s)
+      end || raise(ActiveRecord::RecordNotFound)
+
+      parent.cycle_analytics_stages.build(raw_stage)
+    end
+  end
+end
diff --git a/app/services/cycle_analytics/stage_list_service.rb b/app/services/cycle_analytics/stage_list_service.rb
new file mode 100644
index 00000000000..b5dc1a66fa9
--- /dev/null
+++ b/app/services/cycle_analytics/stage_list_service.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module CycleAnalytics
+  class StageListService
+    def initialize(parent:, allowed_to_customize_stages: false)
+      @parent = parent
+      @allowed_to_customize_stages = allowed_to_customize_stages
+    end
+
+    def execute
+      if allowed_to_customize_stages
+        raise NotImplementedError # will be implemented in EE
+      else
+        build_default_stages
+      end
+    end
+
+    private
+
+    attr_reader :parent, :allowed_to_customize_stages
+
+    def build_default_stages
+      Gitlab::CycleAnalytics::DefaultStages.all.map do |params|
+        parent.cycle_analytics_stages.build(params)
+      end
+    end
+  end
+end
diff --git a/changelogs/unreleased/new-cycle-analytics-stage-backend.yml b/changelogs/unreleased/new-cycle-analytics-stage-backend.yml
new file mode 100644
index 00000000000..677a9d2d82b
--- /dev/null
+++ b/changelogs/unreleased/new-cycle-analytics-stage-backend.yml
@@ -0,0 +1,5 @@
+---
+title: New Cycle Analytics backend
+merge_request: 31535
+author:
+type: changed
diff --git a/config/routes/project.rb b/config/routes/project.rb
index 3113cb172f7..21ccc854bf8 100644
--- a/config/routes/project.rb
+++ b/config/routes/project.rb
@@ -427,9 +427,14 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
         end
       end
 
-      resource :cycle_analytics, only: [:show]
+      resource :cycle_analytics, only: [:show] do
+        get 'median/:stage_id' => 'cycle_analytics#median'
+      end
 
       namespace :cycle_analytics do
+        resources :records, only: [] do
+          get '/:stage_id' => 'records#index', on: :collection
+        end
         scope :events, controller: 'events' do
           get :issue
           get :plan
diff --git a/db/migrate/20190716144222_create_cycle_analytics_project_stages.rb b/db/migrate/20190716144222_create_cycle_analytics_project_stages.rb
new file mode 100644
index 00000000000..83a7dddedf0
--- /dev/null
+++ b/db/migrate/20190716144222_create_cycle_analytics_project_stages.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class CreateCycleAnalyticsProjectStages < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+  INDEX_NAME = 'index_cycle_analytics_stages_on_project_id_and_name'
+
+  def change
+    create_table :cycle_analytics_project_stages do |t|
+      t.references :project, null: false, foreign_key: { to_table: :projects, on_delete: :cascade }
+      t.string :name, null: false
+      t.boolean :hidden, default: false, null: false
+      t.boolean :custom, default: true, null: false
+      t.integer :relative_position
+      t.integer :start_event_identifier, null: false
+      t.integer :end_event_identifier, null: false
+      t.references :start_event_label, foreign_key: { to_table: :labels, on_delete: :cascade }
+      t.references :end_event_label, foreign_key: { to_table: :labels, on_delete: :cascade }
+
+      t.timestamps_with_timezone
+    end
+
+    add_index :cycle_analytics_project_stages, [:project_id, :name], unique: true, name: INDEX_NAME
+    add_index :cycle_analytics_project_stages, [:relative_position]
+  end
+end
diff --git a/db/migrate/20190729062536_create_cycle_analytics_group_stages.rb b/db/migrate/20190729062536_create_cycle_analytics_group_stages.rb
new file mode 100644
index 00000000000..9eed96c2be0
--- /dev/null
+++ b/db/migrate/20190729062536_create_cycle_analytics_group_stages.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class CreateCycleAnalyticsGroupStages < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+  INDEX_NAME = 'index_cycle_analytics_stages_on_group_id_and_name'
+
+  def change
+    create_table :cycle_analytics_group_stages do |t|
+      t.references :group, null: false, foreign_key: { to_table: :namespaces, on_delete: :cascade }
+      t.string :name, null: false
+      t.boolean :hidden, default: false, null: false
+      t.boolean :custom, default: true, null: false
+      t.integer :relative_position
+      t.integer :start_event_identifier, null: false
+      t.integer :end_event_identifier, null: false
+      t.references :start_event_label, foreign_key: { to_table: :labels, on_delete: :cascade }
+      t.references :end_event_label, foreign_key: { to_table: :labels, on_delete: :cascade }
+
+      t.timestamps_with_timezone
+    end
+
+    add_index :cycle_analytics_group_stages, [:group_id, :name], unique: true, name: INDEX_NAME
+    add_index :cycle_analytics_group_stages, [:relative_position]
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 828e36aa96c..8f8b38e9c0a 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -1038,6 +1038,44 @@ ActiveRecord::Schema.define(version: 2019_08_02_235445) do
     t.float "percentage_service_desk_issues", default: 0.0, null: false
   end
 
+  create_table "cycle_analytics_group_stages", force: :cascade do |t|
+    t.bigint "group_id", null: false
+    t.string "name", null: false
+    t.boolean "hidden", default: false, null: false
+    t.boolean "custom", default: true, null: false
+    t.integer "relative_position"
+    t.integer "start_event_identifier", null: false
+    t.integer "end_event_identifier", null: false
+    t.bigint "start_event_label_id"
+    t.bigint "end_event_label_id"
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.index ["end_event_label_id"], name: "index_cycle_analytics_group_stages_on_end_event_label_id"
+    t.index ["group_id", "name"], name: "index_cycle_analytics_stages_on_group_id_and_name", unique: true
+    t.index ["group_id"], name: "index_cycle_analytics_group_stages_on_group_id"
+    t.index ["relative_position"], name: "index_cycle_analytics_group_stages_on_relative_position"
+    t.index ["start_event_label_id"], name: "index_cycle_analytics_group_stages_on_start_event_label_id"
+  end
+
+  create_table "cycle_analytics_project_stages", force: :cascade do |t|
+    t.bigint "project_id", null: false
+    t.string "name", null: false
+    t.boolean "hidden", default: false, null: false
+    t.boolean "custom", default: true, null: false
+    t.integer "relative_position"
+    t.integer "start_event_identifier", null: false
+    t.integer "end_event_identifier", null: false
+    t.bigint "start_event_label_id"
+    t.bigint "end_event_label_id"
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.index ["end_event_label_id"], name: "index_cycle_analytics_project_stages_on_end_event_label_id"
+    t.index ["project_id", "name"], name: "index_cycle_analytics_stages_on_project_id_and_name", unique: true
+    t.index ["project_id"], name: "index_cycle_analytics_project_stages_on_project_id"
+    t.index ["relative_position"], name: "index_cycle_analytics_project_stages_on_relative_position"
+    t.index ["start_event_label_id"], name: "index_cycle_analytics_project_stages_on_start_event_label_id"
+  end
+
   create_table "dependency_proxy_blobs", id: :serial, force: :cascade do |t|
     t.datetime_with_timezone "created_at", null: false
     t.text "file", null: false
@@ -3718,6 +3756,12 @@ ActiveRecord::Schema.define(version: 2019_08_02_235445) do
   add_foreign_key "clusters_kubernetes_namespaces", "environments", on_delete: :nullify
   add_foreign_key "clusters_kubernetes_namespaces", "projects", on_delete: :nullify
   add_foreign_key "container_repositories", "projects"
+  add_foreign_key "cycle_analytics_group_stages", "labels", column: "end_event_label_id", on_delete: :cascade
+  add_foreign_key "cycle_analytics_group_stages", "labels", column: "start_event_label_id", on_delete: :cascade
+  add_foreign_key "cycle_analytics_group_stages", "namespaces", column: "group_id", on_delete: :cascade
+  add_foreign_key "cycle_analytics_project_stages", "labels", column: "end_event_label_id", on_delete: :cascade
+  add_foreign_key "cycle_analytics_project_stages", "labels", column: "start_event_label_id", on_delete: :cascade
+  add_foreign_key "cycle_analytics_project_stages", "projects", on_delete: :cascade
   add_foreign_key "dependency_proxy_blobs", "namespaces", column: "group_id", name: "fk_db58bbc5d7", on_delete: :cascade
   add_foreign_key "dependency_proxy_group_settings", "namespaces", column: "group_id", name: "fk_616ddd680a", on_delete: :cascade
   add_foreign_key "deploy_keys_projects", "projects", name: "fk_58a901ca7e", on_delete: :cascade
diff --git a/lib/gitlab/cycle_analytics/base_query_builder.rb b/lib/gitlab/cycle_analytics/base_query_builder.rb
new file mode 100644
index 00000000000..7e10196c8ae
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/base_query_builder.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    class BaseQueryBuilder
+      include Gitlab::CycleAnalytics::MetricsTables
+      include StageQueryHelpers
+
+      delegate :subject_model, to: :stage
+
+      PROJECT_QUERY_RULES = {
+        Project => {
+          Issue => ->(query) {
+            query.join(projects_table).on(issue_table[:project_id].eq(projects_table[:id]))
+              .where(issue_table[:project_id].eq(stage.parent.id))
+          },
+          MergeRequest => ->(query) {
+            query.join(projects_table).on(mr_table[:target_project_id].eq(projects_table[:id]))
+              .where(mr_table[:target_project_id].eq(stage.parent.id))
+          }
+        }
+      }.freeze
+
+      def initialize(stage:, params: {})
+        @stage = stage
+        @params = params
+      end
+
+      def apply
+        query = model_arel_table
+        query = filter_by_parent_model(query)
+        query = filter_by_time_range(query)
+        query = query.join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id]))
+        query = stage.start_event.apply_query_customization(query)
+        query = stage.end_event.apply_query_customization(query)
+        query = query.where(duration.gt(zero_interval))
+        query.where(routes_table[:source_type].eq('Namespace'))
+      end
+
+      private
+
+      attr_reader :stage, :params
+
+      def filter_by_parent_model(query)
+        instance_exec(query, &query_rules.fetch(stage.parent.class).fetch(subject_model))
+      end
+
+      def filter_by_time_range(query)
+        from = params[:from] || 30.days.ago
+        to = params[:to] || nil
+
+        query = query.where(model_arel_table[:created_at].gteq(from))
+        query = query.where(model_arel_table[:created_at].lteq(to)) if to
+        query
+      end
+
+      def model_arel_table
+        subject_model.arel_table
+      end
+
+      # EE will override this to include Group rules
+      def query_rules
+        PROJECT_QUERY_RULES
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/build_records_fetcher.rb b/lib/gitlab/cycle_analytics/build_records_fetcher.rb
new file mode 100644
index 00000000000..c57833e8b27
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/build_records_fetcher.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    class BuildRecordsFetcher
+      include StageQueryHelpers
+      include Gitlab::CycleAnalytics::MetricsTables
+
+      def initialize(stage, query)
+        @stage = stage
+        @query = query
+      end
+
+      def serialized_records
+        AnalyticsBuildSerializer.new.represent(records.map { |e| e['build'] })
+      end
+
+      attr_reader :stage, :query
+
+      def records
+        q = query
+          .join(build_table).on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id]))
+          .project(*projections, round_duration_to_seconds.as('total_time'))
+        result = execute_query(q).to_a
+        Updater.update!(result, from: 'id', to: 'build', klass: ::Ci::Build)
+
+        result
+      end
+
+      def projections
+        [
+          build_table[:id]
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/data_collector.rb b/lib/gitlab/cycle_analytics/data_collector.rb
new file mode 100644
index 00000000000..544fd17f3c5
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/data_collector.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    # Arguments:
+    #   stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::GroupStage
+    #   params:
+    #     current_user: an instance of User
+    #     from: DateTime
+    #     to: DateTime
+    #     project_ids: array of integers, optional, filtering projects within a group, used when the stage is a CycleAnalytics::GroupStage
+    class DataCollector
+      def initialize(stage, params = {})
+        @stage = stage
+        @params = params
+      end
+
+      def records_fetcher
+        RecordsFetcher.new(stage: stage, query: query, params: params)
+      end
+
+      def median
+        Median.new(stage: stage, query: query)
+      end
+
+      private
+
+      attr_reader :stage, :params
+
+      def query
+        BaseQueryBuilder.new(stage: stage, params: params).apply
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/default_stages.rb b/lib/gitlab/cycle_analytics/default_stages.rb
new file mode 100644
index 00000000000..d6edf3cb684
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/default_stages.rb
@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module DefaultStages
+      def self.all
+        [
+          params_for_issue_stage,
+          params_for_plan_stage,
+          params_for_code_stage,
+          params_for_test_stage,
+          params_for_review_stage,
+          params_for_staging_stage,
+          params_for_production_stage
+        ]
+      end
+
+      def self.params_for_issue_stage
+        {
+          name: 'issue',
+          custom: false,
+          relative_position: 1,
+          start_event_identifier: :issue_created,
+          end_event_identifier: :issue_stage_end
+        }
+      end
+
+      def self.params_for_plan_stage
+        {
+          name: 'plan',
+          custom: false,
+          relative_position: 2,
+          start_event_identifier: :plan_stage_start,
+          end_event_identifier: :issue_first_mentioned_in_commit
+        }
+      end
+
+      def self.params_for_code_stage
+        {
+          name: 'code',
+          custom: false,
+          relative_position: 3,
+          start_event_identifier: :code_stage_start,
+          end_event_identifier: :merge_request_created
+        }
+      end
+
+      def self.params_for_test_stage
+        {
+          name: 'test',
+          custom: false,
+          relative_position: 4,
+          start_event_identifier: :merge_request_last_build_started,
+          end_event_identifier: :merge_request_last_build_finished
+        }
+      end
+
+      def self.params_for_review_stage
+        {
+          name: 'review',
+          custom: false,
+          relative_position: 5,
+          start_event_identifier: :merge_request_created,
+          end_event_identifier: :merge_request_merged
+        }
+      end
+
+      def self.params_for_staging_stage
+        {
+          name: 'staging',
+          custom: false,
+          relative_position: 6,
+          start_event_identifier: :merge_request_merged,
+          end_event_identifier: :merge_request_first_deployed_to_production
+        }
+      end
+
+      def self.params_for_production_stage
+        {
+          name: 'production',
+          custom: false,
+          relative_position: 7,
+          start_event_identifier: :merge_request_merged,
+          end_event_identifier: :merge_request_first_deployed_to_production
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/median.rb b/lib/gitlab/cycle_analytics/median.rb
new file mode 100644
index 00000000000..d8a9cb7f277
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/median.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    class Median
+      include StageQueryHelpers
+
+      def initialize(stage:, query:)
+        @stage = stage
+        @query = query
+      end
+
+      def seconds
+        result = execute_query(query.project(median_duration_in_seconds.as('median'))).first || {}
+        result['median'] ? result['median'].to_i : nil
+      end
+
+      private
+
+      attr_reader :stage, :query
+
+      def percentile_cont
+        percentile_disc_ordering = Arel::Nodes::UnaryOperation.new(Arel::Nodes::SqlLiteral.new('ORDER BY'), duration)
+        Arel::Nodes::NamedFunction.new(
+          'percentile_cont(0.5) WITHIN GROUP',
+          [percentile_disc_ordering]
+        )
+      end
+
+      def median_duration_in_seconds
+        Arel::Nodes::Extract.new(percentile_cont, :epoch)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/records_fetcher.rb b/lib/gitlab/cycle_analytics/records_fetcher.rb
new file mode 100644
index 00000000000..bdc5dc8776a
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/records_fetcher.rb
@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    class RecordsFetcher
+      include StageQueryHelpers
+      include Gitlab::CycleAnalytics::MetricsTables
+
+      MAX_RECORDS = 50
+
+      FINDER_CLASS_MAPPING = {
+        Issue => IssuesFinder,
+        MergeRequest => MergeRequestsFinder
+      }.freeze
+
+      SERIALIZER_CLASS_MAPPING = {
+        Issue => AnalyticsIssueSerializer,
+        MergeRequest => AnalyticsMergeRequestSerializer
+      }.freeze
+
+      delegate :subject_model, to: :stage
+
+      def initialize(stage:, query:, params: {})
+        @stage = stage
+        @query = query
+        @params = params
+      end
+
+      def serialized_records
+        # Test and Staging stages should load Ci::Build records
+        if default_test_stage? || default_staging_stage?
+          BuildRecordsFetcher.new(stage, query, params).serialized_records
+        else
+          q = query
+            .join(finder_arel_query).on(finder_arel_query[:id].eq(subject_model.arel_table[:id]))
+            .order(stage.end_event.timestamp_projection.desc)
+            .take(MAX_RECORDS)
+          q = q.project(*projection_mapping[subject_model], round_duration_to_seconds.as('total_time'))
+          execute_query(q).to_a.map do |item|
+            SERIALIZER_CLASS_MAPPING.fetch(subject_model).new.represent(item)
+          end
+        end
+      end
+
+      # Casting ActiveRecord::Relation returned by the finder class to Arel so it can be joined with the main Arel query
+      def finder_arel_query
+        @finder_arel_query ||= begin
+                                 ar_relation = FINDER_CLASS_MAPPING.fetch(subject_model)
+                                   .new(params[:current_user], finder_params)
+                                   .execute
+                                 ar_relation = ar_relation.select(subject_model.arel_table[:id])
+                                 ar_relation.arel.as('finder_results')
+                               end
+      end
+
+      private
+
+      attr_reader :stage, :query, :params
+
+      def projection_mapping
+        {
+          Issue => [
+            issue_table[:title],
+            issue_table[:iid],
+            issue_table[:id],
+            issue_table[:created_at],
+            issue_table[:author_id],
+            projects_table[:path].as('project_path'),
+            routes_table[:path].as('namespace_path')
+          ],
+          MergeRequest => [
+            mr_table[:title],
+            mr_table[:iid],
+            mr_table[:id],
+            mr_table[:created_at],
+            mr_table[:state],
+            mr_table[:author_id],
+            projects_table[:path].as('project_path'),
+            routes_table[:path].as('namespace_path')
+          ]
+        }
+      end
+
+      # EE will override this to include Group rules
+      def finder_params
+        {
+          Project => { project_id: stage.parent.id }
+        }.fetch(stage.parent.class)
+      end
+
+      def default_test_stage?
+        stage.matches_with_stage_params?(Gitlab::CycleAnalytics::DefaultStages.params_for_test_stage)
+      end
+
+      def default_staging_stage?
+        stage.matches_with_stage_params?(Gitlab::CycleAnalytics::DefaultStages.params_for_staging_stage)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_events.rb b/lib/gitlab/cycle_analytics/stage_events.rb
new file mode 100644
index 00000000000..c5c37df56d6
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_events.rb
@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module StageEvents
+      # Convention:
+      # Issue: < 100
+      # MergeRequest: >= 100 && < 1000
+      # Custom events for default stages: >= 1000
+      ENUM_MAPPING = {
+        StageEvents::IssueCreated => 1,
+        StageEvents::IssueFirstMentionedInCommit => 2,
+        StageEvents::MergeRequestCreated => 100,
+        StageEvents::MergeRequestFirstDeployedToProduction => 101,
+        StageEvents::MergeRequestLastBuildFinished => 102,
+        StageEvents::MergeRequestLastBuildStarted => 103,
+        StageEvents::MergeRequestMerged => 104,
+        StageEvents::CodeStageStart => 1000,
+        StageEvents::IssueStageEnd => 1001,
+        StageEvents::PlanStageStart => 1002
+      }.freeze
+
+      EVENTS = ENUM_MAPPING.keys.freeze
+
+      # Defines which start_event and end_event pairs are allowed
+      PAIRING_RULES = {
+        StageEvents::PlanStageStart => [
+          StageEvents::IssueFirstMentionedInCommit
+        ],
+        StageEvents::CodeStageStart => [
+          StageEvents::MergeRequestCreated
+        ],
+        StageEvents::IssueCreated => [
+          StageEvents::IssueStageEnd
+        ],
+        StageEvents::MergeRequestCreated => [
+          StageEvents::MergeRequestMerged
+        ],
+        StageEvents::MergeRequestLastBuildStarted => [
+          StageEvents::MergeRequestLastBuildFinished
+        ],
+        StageEvents::MergeRequestMerged => [
+          StageEvents::MergeRequestFirstDeployedToProduction
+        ]
+      }.freeze
+
+      def [](identifier)
+        events.find { |e| e.identifier.to_s.eql?(identifier.to_s) } || raise(KeyError)
+      end
+
+      # hash for AR enum: identifier => number
+      def to_enum
+        ENUM_MAPPING.each_with_object({}) { |(k, v), hash| hash[k.identifier] = v }
+      end
+
+      # will be overridden in EE with custom events
+      def pairing_rules
+        PAIRING_RULES
+      end
+
+      # will be overridden in EE with custom events
+      def events
+        EVENTS
+      end
+
+      module_function :[], :to_enum, :pairing_rules, :events
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/cycle_analytics/stage_events/code_stage_start.rb
new file mode 100644
index 00000000000..99882f1f839
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_events/code_stage_start.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module StageEvents
+      class CodeStageStart < SimpleStageEvent
+        def self.name
+          s_("CycleAnalyticsEvent|Issue first mentioned in a commit")
+        end
+
+        def self.identifier
+          :code_stage_start
+        end
+
+        def object_type
+          MergeRequest
+        end
+
+        def timestamp_projection
+          issue_metrics_table[:first_mentioned_in_commit_at]
+        end
+
+        def apply_query_customization(query)
+          q = inner_join(query, mr_closing_issues_table[:merge_request_id])
+          q = inner_join(q, issue_metrics_table[:issue_id], mr_closing_issues_table[:issue_id])
+          q
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_events/issue_created.rb b/lib/gitlab/cycle_analytics/stage_events/issue_created.rb
new file mode 100644
index 00000000000..c57fbb58eeb
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_events/issue_created.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module StageEvents
+      class IssueCreated < SimpleStageEvent
+        def self.name
+          s_("CycleAnalyticsEvent|Issue created")
+        end
+
+        def self.identifier
+          :issue_created
+        end
+
+        def object_type
+          Issue
+        end
+
+        def timestamp_projection
+          issue_table[:created_at]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb b/lib/gitlab/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
new file mode 100644
index 00000000000..78d78e65266
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module StageEvents
+      class IssueFirstMentionedInCommit < SimpleStageEvent
+        def self.name
+          s_("CycleAnalyticsEvent|Issue first mentioned in a commit")
+        end
+
+        def self.identifier
+          :issue_first_mentioned_in_commit
+        end
+
+        def object_type
+          Issue
+        end
+
+        def timestamp_projection
+          issue_metrics_table[:first_mentioned_in_commit_at]
+        end
+
+        def apply_query_customization(query)
+          inner_join(query, issue_metrics_table[:issue_id])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_events/issue_stage_end.rb b/lib/gitlab/cycle_analytics/stage_events/issue_stage_end.rb
new file mode 100644
index 00000000000..f227b073173
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_events/issue_stage_end.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module StageEvents
+      class IssueStageEnd < SimpleStageEvent
+        def self.name
+          s_("CycleAnalyticsEvent|Issue first associated with a milestone or issue first added to a board")
+        end
+
+        def self.identifier
+          :issue_stage_end
+        end
+
+        def object_type
+          Issue
+        end
+
+        def timestamp_projection
+          Arel::Nodes::NamedFunction.new('COALESCE', [
+            issue_metrics_table[:first_associated_with_milestone_at],
+            issue_metrics_table[:first_added_to_board_at]
+          ])
+        end
+
+        def apply_query_customization(query)
+          inner_join(query, issue_metrics_table[:issue_id])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_events/merge_request_created.rb b/lib/gitlab/cycle_analytics/stage_events/merge_request_created.rb
new file mode 100644
index 00000000000..e2237e2a7a4
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_events/merge_request_created.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module StageEvents
+      class MergeRequestCreated < SimpleStageEvent
+        def self.name
+          s_("CycleAnalyticsEvent|Merge request created")
+        end
+
+        def self.identifier
+          :merge_request_created
+        end
+
+        def object_type
+          MergeRequest
+        end
+
+        def timestamp_projection
+          mr_table[:created_at]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb b/lib/gitlab/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
new file mode 100644
index 00000000000..16d5992a916
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module StageEvents
+      class MergeRequestFirstDeployedToProduction < SimpleStageEvent
+        def self.name
+          s_("CycleAnalyticsEvent|Merge request first deployed to production")
+        end
+
+        def self.identifier
+          :merge_request_first_deployed_to_production
+        end
+
+        def object_type
+          MergeRequest
+        end
+
+        def timestamp_projection
+          mr_metrics_table[:first_deployed_to_production_at]
+        end
+
+        def apply_query_customization(query)
+          inner_join(query, mr_metrics_table[:merge_request_id])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_events/merge_request_last_build_finished.rb b/lib/gitlab/cycle_analytics/stage_events/merge_request_last_build_finished.rb
new file mode 100644
index 00000000000..7e0656fb5d5
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_events/merge_request_last_build_finished.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module StageEvents
+      class MergeRequestLastBuildFinished < SimpleStageEvent
+        def self.name
+          s_("CycleAnalyticsEvent|Merge request last build finish time")
+        end
+
+        def self.identifier
+          :merge_request_last_build_finished
+        end
+
+        def object_type
+          MergeRequest
+        end
+
+        def timestamp_projection
+          mr_metrics_table[:latest_build_finished_at]
+        end
+
+        def apply_query_customization(query)
+          inner_join(query, mr_metrics_table[:merge_request_id])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_events/merge_request_last_build_started.rb b/lib/gitlab/cycle_analytics/stage_events/merge_request_last_build_started.rb
new file mode 100644
index 00000000000..ae2a01fc421
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_events/merge_request_last_build_started.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module StageEvents
+      class MergeRequestLastBuildStarted < SimpleStageEvent
+        def self.name
+          s_("CycleAnalyticsEvent|Merge request last build start time")
+        end
+
+        def self.identifier
+          :merge_request_last_build_started
+        end
+
+        def object_type
+          MergeRequest
+        end
+
+        def timestamp_projection
+          mr_metrics_table[:latest_build_started_at]
+        end
+
+        def apply_query_customization(query)
+          inner_join(query, mr_metrics_table[:merge_request_id])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_events/merge_request_merged.rb b/lib/gitlab/cycle_analytics/stage_events/merge_request_merged.rb
new file mode 100644
index 00000000000..fba8791ab2d
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_events/merge_request_merged.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module StageEvents
+      class MergeRequestMerged < SimpleStageEvent
+        def self.name
+          s_("CycleAnalyticsEvent|Merge request merged")
+        end
+
+        def self.identifier
+          :merge_request_merged
+        end
+
+        def object_type
+          MergeRequest
+        end
+
+        def timestamp_projection
+          mr_metrics_table[:merged_at]
+        end
+
+        def apply_query_customization(query)
+          inner_join(query, mr_metrics_table[:merge_request_id])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/cycle_analytics/stage_events/plan_stage_start.rb
new file mode 100644
index 00000000000..34462cb4bb0
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_events/plan_stage_start.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module StageEvents
+      class PlanStageStart < SimpleStageEvent
+        def self.name
+          s_("CycleAnalyticsEvent|Issue first associated with a milestone or issue first added to a board")
+        end
+
+        def self.identifier
+          :plan_stage_start
+        end
+
+        def object_type
+          Issue
+        end
+
+        def timestamp_projection
+          Arel::Nodes::NamedFunction.new('COALESCE', [
+            issue_metrics_table[:first_associated_with_milestone_at],
+            issue_metrics_table[:first_added_to_board_at]
+          ])
+        end
+
+        def apply_query_customization(query)
+          q = inner_join(query, issue_metrics_table[:issue_id])
+          q.where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))).where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil))
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_events/simple_stage_event.rb b/lib/gitlab/cycle_analytics/stage_events/simple_stage_event.rb
new file mode 100644
index 00000000000..9764a18ccdc
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_events/simple_stage_event.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module StageEvents
+      # Represents a simple event that usually refers to one database column and does not require additional user input
+      class SimpleStageEvent < StageEvent
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/cycle_analytics/stage_events/stage_event.rb
new file mode 100644
index 00000000000..7bcaa8016d1
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_events/stage_event.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module StageEvents
+      # Base class for expressing an event.
+      class StageEvent
+        include Gitlab::CycleAnalytics::MetricsTables
+
+        def initialize(params)
+          @params = params
+        end
+
+        def self.name
+          raise NotImplementedError
+        end
+
+        # Each StageEvent must expose a timestamp or a timestamp like expression in order to build a range query.
+        # Example: get me all the Issue records between start event end end event
+        def timestamp_projection
+          raise NotImplementedError
+        end
+
+        # Optionally a StageEvent may apply additional filtering or join other tables on the base query.
+        def apply_query_customization(query)
+          query
+        end
+
+        private
+
+        attr_reader :params
+
+        # inner joins a table by a column, table won't be joined if it's already joined
+        #
+        # Example:
+        #
+        # INNER JOIN "table for right_column" ON left_column = right_column;
+        #
+        # Attributes:
+        #
+        # * +query+ - Arel query
+        # * +right_column+ - Column on the right side
+        # * +left_column+ - Defaults to the 'id' column
+        def inner_join(query, right_column, left_column = object_type.arel_table[:id])
+          return query if table_already_inner_joined?(query, right_column.relation)
+
+          query
+            .join(right_column.relation)
+            .on(left_column.eq(right_column))
+        end
+
+        def table_already_inner_joined?(query, table)
+          return false unless query.is_a?(Arel::SelectManager)
+
+          Array(query.source.right).any? { |join| join.left.eql?(table) }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/cycle_analytics/stage_query_helpers.rb
new file mode 100644
index 00000000000..c567cd7dc87
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_query_helpers.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module StageQueryHelpers
+      def execute_query(query)
+        # Extract raw sql and variable bindings from arel query
+        sql, binds = ActiveRecord::Base.connection.send(:to_sql_and_binds, query) # rubocop:disable GitlabSecurity/PublicSend
+
+        ActiveRecord::Base.connection.exec_query(sql, nil, binds).to_a
+      end
+
+      def zero_interval
+        Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")])
+      end
+
+      def round_duration_to_seconds
+        Arel::Nodes::Extract.new(duration, :epoch)
+      end
+
+      def duration
+        Arel::Nodes::Subtraction.new(
+          stage.end_event.timestamp_projection,
+          stage.start_event.timestamp_projection
+        )
+      end
+    end
+  end
+end
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 86317dd887f..5fa5d39e38c 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -3525,6 +3525,30 @@ msgstr ""
 msgid "Cycle Analytics gives an overview of how much time it takes to go from idea to production in your project."
 msgstr ""
 
+msgid "CycleAnalyticsEvent|Issue created"
+msgstr ""
+
+msgid "CycleAnalyticsEvent|Issue first associated with a milestone or issue first added to a board"
+msgstr ""
+
+msgid "CycleAnalyticsEvent|Issue first mentioned in a commit"
+msgstr ""
+
+msgid "CycleAnalyticsEvent|Merge request created"
+msgstr ""
+
+msgid "CycleAnalyticsEvent|Merge request first deployed to production"
+msgstr ""
+
+msgid "CycleAnalyticsEvent|Merge request last build finish time"
+msgstr ""
+
+msgid "CycleAnalyticsEvent|Merge request last build start time"
+msgstr ""
+
+msgid "CycleAnalyticsEvent|Merge request merged"
+msgstr ""
+
 msgid "CycleAnalyticsStage|Code"
 msgstr ""
 
diff --git a/spec/factories/cycle_analytics/project_stages.rb b/spec/factories/cycle_analytics/project_stages.rb
new file mode 100644
index 00000000000..a20ecd34f03
--- /dev/null
+++ b/spec/factories/cycle_analytics/project_stages.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :cycle_analytics_project_stage, class: CycleAnalytics::ProjectStage do
+    project
+    sequence(:name) { |n| "Stage ##{n}" }
+    hidden { false }
+    issue_stage
+
+    trait :issue_stage do
+      start_event_identifier { Gitlab::CycleAnalytics::StageEvents::IssueCreated.identifier }
+      end_event_identifier { Gitlab::CycleAnalytics::StageEvents::IssueStageEnd.identifier }
+    end
+  end
+end
diff --git a/spec/lib/gitlab/cycle_analytics/code_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/code_stage_spec.rb
index dd1d9ac0f16..d6b35bb872c 100644
--- a/spec/lib/gitlab/cycle_analytics/code_stage_spec.rb
+++ b/spec/lib/gitlab/cycle_analytics/code_stage_spec.rb
@@ -12,7 +12,8 @@ describe Gitlab::CycleAnalytics::CodeStage do
   let(:issue_3) { create(:issue, project: project, created_at: 60.minutes.ago) }
   let(:mr_1) { create(:merge_request, source_project: project, created_at: 15.minutes.ago) }
   let(:mr_2) { create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'A') }
-  let(:stage) { described_class.new(options: { from: 2.days.ago, current_user: project.creator, project: project }) }
+  let(:from) { 2.days.ago }
+  let(:stage) { described_class.new(options: { from: from, current_user: project.creator, project: project }) }
 
   before do
     issue_1.metrics.update!(first_associated_with_milestone_at: 60.minutes.ago, first_mentioned_in_commit_at: 45.minutes.ago)
@@ -33,6 +34,11 @@ describe Gitlab::CycleAnalytics::CodeStage do
     it 'counts median from issues with metrics' do
       expect(stage.project_median).to eq(ISSUES_MEDIAN)
     end
+
+    it_behaves_like 'using Gitlab::CycleAnalytics::DataCollector as backend' do
+      let(:expected_record_count) { 2 }
+      let(:expected_record_titles) { [mr_2.title, mr_1.title] }
+    end
   end
 
   describe '#events' do
diff --git a/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb
index 4dd21239cde..cc0e8adb4b4 100644
--- a/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb
+++ b/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb
@@ -10,7 +10,8 @@ describe Gitlab::CycleAnalytics::IssueStage do
   let(:issue_2) { create(:issue, project: project, created_at: 60.minutes.ago) }
   let(:issue_3) { create(:issue, project: project, created_at: 30.minutes.ago) }
   let!(:issue_without_milestone) { create(:issue, project: project, created_at: 1.minute.ago) }
-  let(:stage) { described_class.new(options: { from: 2.days.ago, current_user: project.creator, project: project }) }
+  let(:from) { 2.days.ago }
+  let(:stage) { described_class.new(options: { from: from, current_user: project.creator, project: project }) }
 
   before do
     issue_1.metrics.update!(first_associated_with_milestone_at: 60.minutes.ago )
@@ -28,6 +29,11 @@ describe Gitlab::CycleAnalytics::IssueStage do
     it 'counts median from issues with metrics' do
       expect(stage.project_median).to eq(ISSUES_MEDIAN)
     end
+
+    it_behaves_like 'using Gitlab::CycleAnalytics::DataCollector as backend' do
+      let(:expected_record_count) { 3 }
+      let(:expected_record_titles) { [issue_3.title, issue_2.title, issue_1.title] }
+    end
   end
 
   describe '#events' do
diff --git a/spec/lib/gitlab/cycle_analytics/plan_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/plan_stage_spec.rb
index 98d2593de66..66a17cb8d78 100644
--- a/spec/lib/gitlab/cycle_analytics/plan_stage_spec.rb
+++ b/spec/lib/gitlab/cycle_analytics/plan_stage_spec.rb
@@ -10,7 +10,8 @@ describe Gitlab::CycleAnalytics::PlanStage do
   let!(:issue_2) { create(:issue, project: project, created_at: 60.minutes.ago) }
   let!(:issue_3) { create(:issue, project: project, created_at: 30.minutes.ago) }
   let!(:issue_without_milestone) { create(:issue, project: project, created_at: 1.minute.ago) }
-  let(:stage) { described_class.new(options: { from: 2.days.ago, current_user: project.creator, project: project }) }
+  let(:from) { 2.days.ago }
+  let(:stage) { described_class.new(options: { from: from, current_user: project.creator, project: project }) }
 
   before do
     issue_1.metrics.update!(first_associated_with_milestone_at: 60.minutes.ago, first_mentioned_in_commit_at: 10.minutes.ago)
@@ -28,6 +29,11 @@ describe Gitlab::CycleAnalytics::PlanStage do
     it 'counts median from issues with metrics' do
       expect(stage.project_median).to eq(ISSUES_MEDIAN)
     end
+
+    it_behaves_like 'using Gitlab::CycleAnalytics::DataCollector as backend' do
+      let(:expected_record_count) { 2 }
+      let(:expected_record_titles) { [issue_1.title, issue_2.title] }
+    end
   end
 
   describe '#events' do
diff --git a/spec/lib/gitlab/cycle_analytics/records_fetcher_spec.rb b/spec/lib/gitlab/cycle_analytics/records_fetcher_spec.rb
new file mode 100644
index 00000000000..b53ae05db09
--- /dev/null
+++ b/spec/lib/gitlab/cycle_analytics/records_fetcher_spec.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Gitlab::CycleAnalytics::RecordsFetcher do
+  around do |example|
+    Timecop.freeze { example.run }
+  end
+
+  it "respect issue visibility rules, confidential issues won't be listed" do
+    project = create(:project, :empty_repo)
+    user = create(:user)
+
+    project.add_user(user, Gitlab::Access::GUEST)
+
+    issue1 = create(:issue, project: project)
+    issue2 = create(:issue, project: project, confidential: true)
+
+    issue1.metrics.update(first_added_to_board_at: 3.days.ago)
+    issue2.metrics.update(first_added_to_board_at: 3.days.ago)
+
+    issue1.metrics.update!(first_mentioned_in_commit_at: 2.days.ago)
+    issue2.metrics.update!(first_mentioned_in_commit_at: 2.days.ago)
+
+    stage = build(:cycle_analytics_project_stage, {
+      start_event_identifier: :plan_stage_start,
+      end_event_identifier: :issue_first_mentioned_in_commit,
+      project: project
+    })
+
+    data_collector = Gitlab::CycleAnalytics::DataCollector.new(stage, {
+      from: Time.new(2019),
+      current_user: user
+    })
+
+    expect(data_collector.records_fetcher.serialized_records.size).to eq(1)
+  end
+end
diff --git a/spec/lib/gitlab/cycle_analytics/shared_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/shared_stage_spec.rb
index cf95741908f..29373b01603 100644
--- a/spec/lib/gitlab/cycle_analytics/shared_stage_spec.rb
+++ b/spec/lib/gitlab/cycle_analytics/shared_stage_spec.rb
@@ -32,3 +32,19 @@ shared_examples 'base stage' do
     expect(stage.events).not_to be_nil
   end
 end
+
+shared_examples 'using Gitlab::CycleAnalytics::DataCollector as backend' do
+  let(:stage_params) { Gitlab::CycleAnalytics::DefaultStages.send("params_for_#{stage_name}_stage").merge(project: project) }
+  let(:stage) { CycleAnalytics::ProjectStage.new(stage_params) }
+  let(:data_collector) { Gitlab::CycleAnalytics::DataCollector.new(stage, from: from, current_user: project.creator) }
+
+  it 'provides the same median value' do
+    expect(data_collector.median.seconds).to eq(ISSUES_MEDIAN)
+  end
+
+  it 'provides the same event records' do
+    records = data_collector.records_fetcher.serialized_records
+    expect(records.count).to eq(expected_record_count)
+    expect(records.map { |event| event[:title] }).to eq(expected_record_titles)
+  end
+end
diff --git a/spec/lib/gitlab/cycle_analytics/test_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/test_stage_spec.rb
index 9162686d17d..e4ebb9a3b90 100644
--- a/spec/lib/gitlab/cycle_analytics/test_stage_spec.rb
+++ b/spec/lib/gitlab/cycle_analytics/test_stage_spec.rb
@@ -6,20 +6,22 @@ require 'lib/gitlab/cycle_analytics/shared_stage_spec'
 describe Gitlab::CycleAnalytics::TestStage do
   let(:stage_name) { :test }
   let(:project) { create(:project) }
-  let(:stage) { described_class.new(options: { from: 2.days.ago, current_user: project.creator, project: project }) }
+  let(:from) { 2.days.ago }
+  let(:stage) { described_class.new(options: { from: from, current_user: project.creator, project: project }) }
 
   it_behaves_like 'base stage'
 
   describe '#median' do
+    let(:mr_1) { create(:merge_request, :closed, source_project: project, created_at: 60.minutes.ago) }
+    let(:mr_2) { create(:merge_request, :closed, source_project: project, created_at: 40.minutes.ago, source_branch: 'A') }
+    let(:mr_3) { create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'B') }
+    let(:mr_4) { create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'C') }
+    let(:mr_5) { create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'D') }
+
     before do
       issue_1 = create(:issue, project: project, created_at: 90.minutes.ago)
       issue_2 = create(:issue, project: project, created_at: 60.minutes.ago)
       issue_3 = create(:issue, project: project, created_at: 60.minutes.ago)
-      mr_1 = create(:merge_request, :closed, source_project: project, created_at: 60.minutes.ago)
-      mr_2 = create(:merge_request, :closed, source_project: project, created_at: 40.minutes.ago, source_branch: 'A')
-      mr_3 = create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'B')
-      mr_4 = create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'C')
-      mr_5 = create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'D')
       mr_1.metrics.update!(latest_build_started_at: 32.minutes.ago, latest_build_finished_at: 2.minutes.ago)
       mr_2.metrics.update!(latest_build_started_at: 62.minutes.ago, latest_build_finished_at: 32.minutes.ago)
       mr_3.metrics.update!(latest_build_started_at: nil, latest_build_finished_at: nil)
@@ -40,5 +42,10 @@ describe Gitlab::CycleAnalytics::TestStage do
     it 'counts median from issues with metrics' do
       expect(stage.project_median).to eq(ISSUES_MEDIAN)
     end
+
+    it_behaves_like 'using Gitlab::CycleAnalytics::DataCollector as backend' do
+      let(:expected_record_count) { 2 }
+      let(:expected_record_titles) { [mr_1.title, mr_2.title] }
+    end
   end
 end
diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml
index ada8c649ff6..a4d3383e28b 100644
--- a/spec/lib/gitlab/import_export/all_models.yml
+++ b/spec/lib/gitlab/import_export/all_models.yml
@@ -242,6 +242,7 @@ project:
 - cluster_project
 - cluster_ingresses
 - creator
+- cycle_analytics_stages
 - group
 - namespace
 - boards
diff --git a/spec/models/cycle_analytics/project_stage_spec.rb b/spec/models/cycle_analytics/project_stage_spec.rb
new file mode 100644
index 00000000000..1e157a81c9f
--- /dev/null
+++ b/spec/models/cycle_analytics/project_stage_spec.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe CycleAnalytics::ProjectStage do
+  it 'default stages must be valid' do
+    project = create(:project)
+
+    Gitlab::CycleAnalytics::DefaultStages.all.each do |params|
+      stage = described_class.new(params.merge(project: project))
+      expect(stage).to be_valid
+    end
+  end
+
+  it_behaves_like "cycle analytics stage" do
+    let(:parent) { create(:project) }
+  end
+end
diff --git a/spec/serializers/cycle_analytics/stage_decorator_spec.rb b/spec/serializers/cycle_analytics/stage_decorator_spec.rb
new file mode 100644
index 00000000000..ce5c1740492
--- /dev/null
+++ b/spec/serializers/cycle_analytics/stage_decorator_spec.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe CycleAnalytics::StageDecorator do
+  let(:params) { Gitlab::CycleAnalytics::DefaultStages.params_for_issue_stage }
+
+  it 'decorates default stage attributes with localized text' do
+    issue_stage = CycleAnalytics::ProjectStage.new(params)
+
+    decorator = described_class.new(issue_stage)
+
+    expect(decorator.title).to eq(described_class::DEFAULT_STAGE_ATTRIBUTES[:issue][:title].call)
+    expect(decorator.description).to eq(described_class::DEFAULT_STAGE_ATTRIBUTES[:issue][:description].call)
+  end
+
+  describe 'custom stage' do
+    let(:custom_stage) { CycleAnalytics::ProjectStage.new(params) }
+    let(:decorator) { described_class.new(custom_stage) }
+
+    before do
+      params[:name] = 'My Stage'
+    end
+
+    it 'uses name attribute for the title' do
+      expect(decorator.title).to eq(params[:name])
+    end
+
+    it 'uses empty string for description' do
+      expect(decorator.description).to eq('')
+    end
+  end
+
+  it 'infers legend from #subject_model' do
+    issue_stage = CycleAnalytics::ProjectStage.new(params)
+
+    expect(issue_stage.subject_model).to eq(Issue)
+
+    decorator = described_class.new(issue_stage)
+    expect(decorator.legend).to eq(_("Related Issues"))
+  end
+end
diff --git a/spec/services/cycle_analytics/stage_find_service_spec.rb b/spec/services/cycle_analytics/stage_find_service_spec.rb
new file mode 100644
index 00000000000..c072c4f4480
--- /dev/null
+++ b/spec/services/cycle_analytics/stage_find_service_spec.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe CycleAnalytics::StageFindService do
+  it 'finds in-memory default stage' do
+    found_stage = described_class.new(parent: build(:project), id: 'code').execute # code (default) stage name
+
+    code_stage_params = Gitlab::CycleAnalytics::DefaultStages.params_for_code_stage
+    expect(found_stage.name).to eq(code_stage_params[:name])
+    expect(found_stage.start_event_identifier.to_sym).to eq(code_stage_params[:start_event_identifier])
+    expect(found_stage.end_event_identifier.to_sym).to eq(code_stage_params[:end_event_identifier])
+  end
+
+  it 'raises ActiveRecord::RecordNotFound when in-memory default stage cannot be found' do
+    expect do
+      described_class.new(parent: build(:project), id: 'UnknownDefaultStage').execute
+    end.to raise_error(ActiveRecord::RecordNotFound)
+  end
+end
diff --git a/spec/services/cycle_analytics/stage_list_service_spec.rb b/spec/services/cycle_analytics/stage_list_service_spec.rb
new file mode 100644
index 00000000000..17bc1880816
--- /dev/null
+++ b/spec/services/cycle_analytics/stage_list_service_spec.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe CycleAnalytics::StageListService do
+  let(:project) { build(:project, :empty_repo) }
+
+  it 'returns the default stages as in-memory objects if customizable stages are not allowed' do
+    service = described_class.new(parent: project, allowed_to_customize_stages: false)
+
+    stages = service.execute
+
+    stage_names = stages.map(&:name)
+    expect(stage_names).to eq(Gitlab::CycleAnalytics::DefaultStages.all.map { |p| p[:name] })
+
+    stage_ids = stages.map(&:id)
+    expect(stage_ids.all?(&:nil?)).to eq(true)
+  end
+end
diff --git a/spec/support/shared_examples/cycle_analytics_stage_examples.rb b/spec/support/shared_examples/cycle_analytics_stage_examples.rb
new file mode 100644
index 00000000000..f16515fafcb
--- /dev/null
+++ b/spec/support/shared_examples/cycle_analytics_stage_examples.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+shared_examples_for 'cycle analytics stage' do
+  let(:valid_params) do
+    {
+      name: 'My Stage',
+      parent: parent,
+      start_event_identifier: :merge_request_created,
+      end_event_identifier: :merge_request_merged
+    }
+  end
+
+  describe 'validation' do
+    it 'is valid' do
+      expect(described_class.new(valid_params)).to be_valid
+    end
+
+    it 'is invalid when end_event is not allowed for the given start_event' do
+      invalid_params = valid_params.merge(
+        start_event_identifier: :merge_request_merged,
+        end_event_identifier: :merge_request_created
+      )
+      stage = described_class.new(invalid_params)
+
+      expect(stage).not_to be_valid
+      expect(stage.errors.details[:end_event]).to eq([{ error: :not_allowed_for_the_given_start_event }])
+    end
+  end
+
+  describe '#subject_model' do
+    it 'infers the model to be queried from the start event' do
+      stage = described_class.new(valid_params)
+
+      expect(stage.subject_model).to eq(MergeRequest)
+    end
+  end
+end