Merge branch 'ci-commit-as-pipeline' into 'master'

Ci::Commit becomes a Pipeline object

1. Ci::Commit receives context: ref, :tag.
1. One Ci::Commit describes a one Pipeline
1. Pipeline is created from `.gitlab-ci.yml`
1. Pipeline is a ordered group of builds
1. We test MR against Pipeline
1. Pipelines have a separate view (https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3703)
1. Pipeline can be triggered from UI (https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3703)
1. Later we change `Trigger -> TriggerRequest -> Build` to `Trigger -> Pipeline` (future)
1. We add a Pipeline Hook that will be triggered on Pipeline status change  (future)
1. We extend notifications to use `Pipeline Hook` to send summary on pipeline changes (future)

After merging that I'll prepare a separate MR that will unify naming, database columns, table names:
```
Ci::Commit -> Pipeline
Ci::Build -> Build
CommitStatus -> Job
GenericCommitStatus -> ExternalJob

ci_commits -> pipelines
ci_builds -> jobs
```

This MR implements first 5 points.

This is made to solve this issue https://gitlab.com/gitlab-org/gitlab-ce/issues/14149.

See merge request !3653

28 files changed, 331 insertions, 280 deletions

diff --git a/app/controllers/projects/commit_controller.rb b/app/controllers/projects/commit_controller.rb
index 4d64a2d9884..a202cb38692 100644
--- a/app/controllers/projects/commit_controller.rb
+++ b/app/controllers/projects/commit_controller.rb
@@ -38,13 +38,13 @@ class Projects::CommitController < Projects::ApplicationController
   end
 
   def cancel_builds
-    ci_commit.builds.running_or_pending.each(&:cancel)
+    ci_builds.running_or_pending.each(&:cancel)
 
     redirect_back_or_default default: builds_namespace_project_commit_path(project.namespace, project, commit.sha)
   end
 
   def retry_builds
-    ci_commit.builds.latest.failed.each do |build|
+    ci_builds.latest.failed.each do |build|
       if build.retryable?
         Ci::Build.retry(build)
       end
@@ -99,8 +99,12 @@ class Projects::CommitController < Projects::ApplicationController
     @commit ||= @project.commit(params[:id])
   end
 
-  def ci_commit
-    @ci_commit ||= project.ci_commit(commit.sha)
+  def ci_commits
+    @ci_commits ||= project.ci_commits.where(sha: commit.sha)
+  end
+
+  def ci_builds
+    @ci_builds ||= Ci::Build.where(commit: ci_commits)
   end
 
   def define_show_vars
@@ -113,7 +117,8 @@ class Projects::CommitController < Projects::ApplicationController
     @diff_refs = [commit.parent || commit, commit]
     @notes_count = commit.notes.count
 
-    @statuses = ci_commit.statuses if ci_commit
+    @statuses = CommitStatus.where(commit: ci_commits)
+    @builds = Ci::Build.where(commit: ci_commits)
   end
 
   def assign_change_commit_vars(mr_source_branch)
diff --git a/app/controllers/projects/merge_requests_controller.rb b/app/controllers/projects/merge_requests_controller.rb
index 1388ea9d66c..81003c34f59 100644
--- a/app/controllers/projects/merge_requests_controller.rb
+++ b/app/controllers/projects/merge_requests_controller.rb
@@ -321,6 +321,7 @@ class Projects::MergeRequestsController < Projects::ApplicationController
 
   def define_widget_vars
     @ci_commit = @merge_request.ci_commit
+    @ci_commits = [@ci_commit].compact
     closes_issues
   end
 
diff --git a/app/helpers/ci_status_helper.rb b/app/helpers/ci_status_helper.rb
index 8b1575d5e0c..417050b4132 100644
--- a/app/helpers/ci_status_helper.rb
+++ b/app/helpers/ci_status_helper.rb
@@ -4,14 +4,6 @@ module CiStatusHelper
     builds_namespace_project_commit_path(project.namespace, project, ci_commit.sha)
   end
 
-  def ci_status_icon(ci_commit)
-    ci_icon_for_status(ci_commit.status)
-  end
-
-  def ci_status_label(ci_commit)
-    ci_label_for_status(ci_commit.status)
-  end
-
   def ci_status_with_icon(status, target = nil)
     content = ci_icon_for_status(status) + '&nbsp;'.html_safe + ci_label_for_status(status)
     klass = "ci-status ci-#{status}"
@@ -47,10 +39,13 @@ module CiStatusHelper
   end
 
   def render_ci_status(ci_commit, tooltip_placement: 'auto left')
-    link_to ci_status_icon(ci_commit),
+    # TODO: split this method into
+    # - render_commit_status
+    # - render_pipeline_status
+    link_to ci_icon_for_status(ci_commit.status),
       ci_status_path(ci_commit),
       class: "ci-status-link ci-status-icon-#{ci_commit.status.dasherize}",
-      title: "Build #{ci_status_label(ci_commit)}",
+      title: "Build #{ci_label_for_status(ci_commit.status)}",
       data: { toggle: 'tooltip', placement: tooltip_placement }
   end
 
diff --git a/app/helpers/gitlab_routing_helper.rb b/app/helpers/gitlab_routing_helper.rb
index f3fddef01cb..f07eff3fb57 100644
--- a/app/helpers/gitlab_routing_helper.rb
+++ b/app/helpers/gitlab_routing_helper.rb
@@ -25,6 +25,10 @@ module GitlabRoutingHelper
     namespace_project_commits_path(project.namespace, project, @ref || project.repository.root_ref)
   end
 
+  def project_pipelines_path(project, *args)
+    namespace_project_pipelines_path(project.namespace, project, *args)
+  end
+
   def project_builds_path(project, *args)
     namespace_project_builds_path(project.namespace, project, *args)
   end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index d42a65620ff..553cd447971 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -37,8 +37,6 @@
 
 module Ci
   class Build < CommitStatus
-    LAZY_ATTRIBUTES = ['trace']
-
     belongs_to :runner, class_name: 'Ci::Runner'
     belongs_to :trigger_request, class_name: 'Ci::TriggerRequest'
     belongs_to :erased_by, class_name: 'User'
@@ -50,25 +48,17 @@ module Ci
 
     scope :unstarted, ->() { where(runner_id: nil) }
     scope :ignore_failures, ->() { where(allow_failure: false) }
-    scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) }
 
     mount_uploader :artifacts_file, ArtifactUploader
     mount_uploader :artifacts_metadata, ArtifactUploader
 
     acts_as_taggable
 
-    # To prevent db load megabytes of data from trace
-    default_scope -> { select(Ci::Build.columns_without_lazy) }
-
     before_destroy { project }
 
-    class << self
-      def columns_without_lazy
-        (column_names - LAZY_ATTRIBUTES).map do |column_name|
-          "#{table_name}.#{column_name}"
-        end
-      end
+    after_create :execute_hooks
 
+    class << self
       def last_month
         where('created_at > ?', Date.today - 1.month)
       end
@@ -126,12 +116,16 @@ module Ci
     end
 
     def retried?
-      !self.commit.latest_statuses_for_ref(self.ref).include?(self)
+      !self.commit.statuses.latest.include?(self)
+    end
+
+    def retry
+      Ci::Build.retry(self)
     end
 
     def depends_on_builds
       # Get builds of the same type
-      latest_builds = self.commit.builds.similar(self).latest
+      latest_builds = self.commit.builds.latest
 
       # Return builds from previous stages
       latest_builds.where('stage_idx < ?', stage_idx)
diff --git a/app/models/ci/commit.rb b/app/models/ci/commit.rb
index f4cf7034b14..f2667e5476b 100644
--- a/app/models/ci/commit.rb
+++ b/app/models/ci/commit.rb
@@ -19,21 +19,28 @@
 module Ci
   class Commit < ActiveRecord::Base
     extend Ci::Model
+    include Statuseable
 
     belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id
     has_many :statuses, class_name: 'CommitStatus'
     has_many :builds, class_name: 'Ci::Build'
     has_many :trigger_requests, dependent: :destroy, class_name: 'Ci::TriggerRequest'
 
+    delegate :stages, to: :statuses
+
     validates_presence_of :sha
+    validates_presence_of :status
     validate :valid_commit_sha
 
+    # Invalidate object and save if when touched
+    after_touch :update_state
+
     def self.truncate_sha(sha)
       sha[0...8]
     end
 
-    def to_param
-      sha
+    def self.stages
+      CommitStatus.where(commit: all).stages
     end
 
     def project_id
@@ -68,15 +75,20 @@ module Ci
       nil
     end
 
-    def stage
-      running_or_pending = statuses.latest.running_or_pending.ordered
-      running_or_pending.first.try(:stage)
+    def branch?
+      !tag?
+    end
+
+    def retryable?
+      builds.latest.any? do |build|
+        build.failed? && build.retryable?
+      end
     end
 
-    def create_builds(ref, tag, user, trigger_request = nil)
+    def create_builds(user, trigger_request = nil)
       return unless config_processor
       config_processor.stages.any? do |stage|
-        CreateBuildsService.new.execute(self, stage, ref, tag, user, trigger_request, 'success').present?
+        CreateBuildsService.new(self).execute(stage, user, 'success', trigger_request).present?
       end
     end
 
@@ -84,7 +96,7 @@ module Ci
       return unless config_processor
 
       # don't create other builds if this one is retried
-      latest_builds = builds.similar(build).latest
+      latest_builds = builds.latest
       return unless latest_builds.exists?(build.id)
 
       # get list of stages after this build
@@ -92,88 +104,21 @@ module Ci
       next_stages.delete(build.stage)
 
       # get status for all prior builds
-      prior_builds = latest_builds.reject { |other_build| next_stages.include?(other_build.stage) }
-      status = Ci::Status.get_status(prior_builds)
+      prior_builds = latest_builds.where.not(stage: next_stages)
+      prior_status = prior_builds.status
 
       # create builds for next stages based
       next_stages.any? do |stage|
-        CreateBuildsService.new.execute(self, stage, build.ref, build.tag, build.user, build.trigger_request, status).present?
+        CreateBuildsService.new(self).execute(stage, build.user, prior_status, build.trigger_request).present?
       end
     end
 
-    def refs
-      statuses.order(:ref).pluck(:ref).uniq
-    end
-
-    def latest_statuses
-      @latest_statuses ||= statuses.latest.to_a
-    end
-
-    def latest_statuses_for_ref(ref)
-      latest_statuses.select { |status| status.ref == ref }
-    end
-
-    def matrix_builds(build = nil)
-      matrix_builds = builds.latest.ordered
-      matrix_builds = matrix_builds.similar(build) if build
-      matrix_builds.to_a
-    end
-
     def retried
       @retried ||= (statuses.order(id: :desc) - statuses.latest)
     end
 
-    def status
-      if yaml_errors.present?
-        return 'failed'
-      end
-
-      @status ||= Ci::Status.get_status(latest_statuses)
-    end
-
-    def pending?
-      status == 'pending'
-    end
-
-    def running?
-      status == 'running'
-    end
-
-    def success?
-      status == 'success'
-    end
-
-    def failed?
-      status == 'failed'
-    end
-
-    def canceled?
-      status == 'canceled'
-    end
-
-    def active?
-      running? || pending?
-    end
-
-    def complete?
-      canceled? || success? || failed?
-    end
-
-    def duration
-      duration_array = statuses.map(&:duration).compact
-      duration_array.reduce(:+).to_i
-    end
-
-    def started_at
-      @started_at ||= statuses.order('started_at ASC').first.try(:started_at)
-    end
-
-    def finished_at
-      @finished_at ||= statuses.order('finished_at DESC').first.try(:finished_at)
-    end
-
     def coverage
-      coverage_array = latest_statuses.map(&:coverage).compact
+      coverage_array = statuses.latest.map(&:coverage).compact
       if coverage_array.size >= 1
         '%.2f' % (coverage_array.reduce(:+) / coverage_array.size)
       end
@@ -181,23 +126,29 @@ module Ci
 
     def config_processor
       return nil unless ci_yaml_file
-      @config_processor ||= Ci::GitlabCiYamlProcessor.new(ci_yaml_file, project.path_with_namespace)
-    rescue Ci::GitlabCiYamlProcessor::ValidationError, Psych::SyntaxError => e
-      save_yaml_error(e.message)
-      nil
-    rescue
-      save_yaml_error("Undefined error")
-      nil
+      return @config_processor if defined?(@config_processor)
+
+      @config_processor ||= begin
+        Ci::GitlabCiYamlProcessor.new(ci_yaml_file, project.path_with_namespace)
+      rescue Ci::GitlabCiYamlProcessor::ValidationError, Psych::SyntaxError => e
+        save_yaml_error(e.message)
+        nil
+      rescue
+        save_yaml_error("Undefined error")
+        nil
+      end
     end
 
     def ci_yaml_file
+      return @ci_yaml_file if defined?(@ci_yaml_file)
+
       @ci_yaml_file ||= begin
         blob = project.repository.blob_at(sha, '.gitlab-ci.yml')
         blob.load_all_data!(project.repository)
         blob.data
+      rescue
+        nil
       end
-    rescue
-      nil
     end
 
     def skip_ci?
@@ -206,10 +157,23 @@ module Ci
 
     private
 
+    def update_state
+      statuses.reload
+      self.status = if yaml_errors.blank?
+                      statuses.latest.status || 'skipped'
+                    else
+                      'failed'
+                    end
+      self.started_at = statuses.started_at
+      self.finished_at = statuses.finished_at
+      self.duration = statuses.latest.duration
+      save
+    end
+
     def save_yaml_error(error)
       return if self.yaml_errors?
       self.yaml_errors = error
-      save
+      update_state
     end
   end
 end
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 6bb018b086f..562c3ed15b2 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -207,12 +207,13 @@ class Commit
     @raw.short_id(7)
   end
 
-  def ci_commit
-    project.ci_commit(sha)
+  def ci_commits
+    @ci_commits ||= project.ci_commits.where(sha: sha)
   end
 
   def status
-    ci_commit.try(:status) || :not_found
+    return @status if defined?(@status)
+    @status ||= ci_commits.status
   end
 
   def revert_branch_name
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 3377a85a55a..aa56314aa16 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -33,30 +33,23 @@
 #
 
 class CommitStatus < ActiveRecord::Base
+  include Statuseable
+
   self.table_name = 'ci_builds'
 
   belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id
-  belongs_to :commit, class_name: 'Ci::Commit'
+  belongs_to :commit, class_name: 'Ci::Commit', touch: true
   belongs_to :user
 
   validates :commit, presence: true
-  validates :status, inclusion: { in: %w(pending running failed success canceled) }
 
   validates_presence_of :name
 
   alias_attribute :author, :user
 
-  scope :running, -> { where(status: 'running') }
-  scope :pending, -> { where(status: 'pending') }
-  scope :success, -> { where(status: 'success') }
-  scope :failed, -> { where(status: 'failed')  }
-  scope :running_or_pending, -> { where(status: [:running, :pending]) }
-  scope :finished, -> { where(status: [:success, :failed, :canceled]) }
-  scope :latest, -> { where(id: unscope(:select).select('max(id)').group(:name, :ref)) }
+  scope :latest, -> { where(id: unscope(:select).select('max(id)').group(:name, :commit_id)) }
   scope :ordered, -> { order(:ref, :stage_idx, :name) }
-  scope :for_ref, ->(ref) { where(ref: ref) }
-
-  AVAILABLE_STATUSES = ['pending', 'running', 'success', 'failed', 'canceled']
+  scope :ignored, -> { where(allow_failure: true, status: [:failed, :canceled]) }
 
   state_machine :status, initial: :pending do
     event :run do
@@ -86,31 +79,24 @@ class CommitStatus < ActiveRecord::Base
     after_transition [:pending, :running] => :success do |commit_status|
       MergeRequests::MergeWhenBuildSucceedsService.new(commit_status.commit.project, nil).trigger(commit_status)
     end
-
-    state :pending, value: 'pending'
-    state :running, value: 'running'
-    state :failed, value: 'failed'
-    state :success, value: 'success'
-    state :canceled, value: 'canceled'
   end
 
-  delegate :sha, :short_sha, to: :commit, prefix: false
+  delegate :sha, :short_sha, to: :commit
 
-  # TODO: this should be removed with all references
   def before_sha
-    Gitlab::Git::BLANK_SHA
+    commit.before_sha || Gitlab::Git::BLANK_SHA
   end
 
-  def started?
-    !pending? && !canceled? && started_at
+  def self.stages
+    order_by = 'max(stage_idx)'
+    group('stage').order(order_by).pluck(:stage, order_by).map(&:first).compact
   end
 
-  def active?
-    running? || pending?
-  end
-
-  def complete?
-    canceled? || success? || failed?
+  def self.stages_status
+    all.stages.inject({}) do |h, stage|
+      h[stage] = all.where(stage: stage).status
+      h
+    end
   end
 
   def ignored?
@@ -118,11 +104,13 @@ class CommitStatus < ActiveRecord::Base
   end
 
   def duration
-    if started_at && finished_at
-      finished_at - started_at
-    elsif started_at
-      Time.now - started_at
-    end
+    duration =
+      if started_at && finished_at
+        finished_at - started_at
+      elsif started_at
+        Time.now - started_at
+      end
+    duration
   end
 
   def stuck?
diff --git a/app/models/concerns/statuseable.rb b/app/models/concerns/statuseable.rb
new file mode 100644
index 00000000000..8a293b7b76e
--- /dev/null
+++ b/app/models/concerns/statuseable.rb
@@ -0,0 +1,81 @@
+module Statuseable
+  extend ActiveSupport::Concern
+
+  AVAILABLE_STATUSES = %w(pending running success failed canceled skipped)
+
+  class_methods do
+    def status_sql
+      builds = all.select('count(*)').to_sql
+      success = all.success.select('count(*)').to_sql
+      ignored = all.ignored.select('count(*)').to_sql if all.respond_to?(:ignored)
+      ignored ||= '0'
+      pending = all.pending.select('count(*)').to_sql
+      running = all.running.select('count(*)').to_sql
+      canceled = all.canceled.select('count(*)').to_sql
+      skipped = all.skipped.select('count(*)').to_sql
+
+      deduce_status = "(CASE
+        WHEN (#{builds})=0 THEN NULL
+        WHEN (#{builds})=(#{success})+(#{ignored}) THEN 'success'
+        WHEN (#{builds})=(#{pending}) THEN 'pending'
+        WHEN (#{builds})=(#{canceled}) THEN 'canceled'
+        WHEN (#{builds})=(#{skipped}) THEN 'skipped'
+        WHEN (#{running})+(#{pending})>0 THEN 'running'
+        ELSE 'failed'
+      END)"
+
+      deduce_status
+    end
+
+    def status
+      all.pluck(self.status_sql).first
+    end
+
+    def duration
+      duration_array = all.map(&:duration).compact
+      duration_array.reduce(:+)
+    end
+
+    def started_at
+      all.minimum(:started_at)
+    end
+
+    def finished_at
+      all.maximum(:finished_at)
+    end
+  end
+
+  included do
+    validates :status, inclusion: { in: AVAILABLE_STATUSES }
+
+    state_machine :status, initial: :pending do
+      state :pending, value: 'pending'
+      state :running, value: 'running'
+      state :failed, value: 'failed'
+      state :success, value: 'success'
+      state :canceled, value: 'canceled'
+      state :skipped, value: 'skipped'
+    end
+
+    scope :running, -> { where(status: 'running') }
+    scope :pending, -> { where(status: 'pending') }
+    scope :success, -> { where(status: 'success') }
+    scope :failed, -> { where(status: 'failed')  }
+    scope :canceled, -> { where(status: 'canceled')  }
+    scope :skipped, -> { where(status: 'skipped')  }
+    scope :running_or_pending, -> { where(status: [:running, :pending]) }
+    scope :finished, -> { where(status: [:success, :failed, :canceled]) }
+  end
+
+  def started?
+    !pending? && !canceled? && started_at
+  end
+
+  def active?
+    running? || pending?
+  end
+
+  def complete?
+    canceled? || success? || failed?
+  end
+end
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index dbecc48485c..d00919c3b0c 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -586,7 +586,7 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def ci_commit
-    @ci_commit ||= source_project.ci_commit(last_commit.id) if last_commit && source_project
+    @ci_commit ||= source_project.ci_commit(last_commit.id, source_branch) if last_commit && source_project
   end
 
   def diff_refs
diff --git a/app/models/project.rb b/app/models/project.rb
index 8f0272d2ce0..7aa21b19e67 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -963,12 +963,12 @@ class Project < ActiveRecord::Base
     !namespace.share_with_group_lock
   end
 
-  def ci_commit(sha)
-    ci_commits.find_by(sha: sha)
+  def ci_commit(sha, ref)
+    ci_commits.order(id: :desc).find_by(sha: sha, ref: ref)
   end
 
-  def ensure_ci_commit(sha)
-    ci_commit(sha) || ci_commits.create(sha: sha)
+  def ensure_ci_commit(sha, ref)
+    ci_commit(sha, ref) || ci_commits.create(sha: sha, ref: ref)
   end
 
   def enable_ci
diff --git a/app/services/ci/create_builds_service.rb b/app/services/ci/create_builds_service.rb
index 2cd51a7610f..18274ce24e2 100644
--- a/app/services/ci/create_builds_service.rb
+++ b/app/services/ci/create_builds_service.rb
@@ -1,7 +1,11 @@
 module Ci
   class CreateBuildsService
-    def execute(commit, stage, ref, tag, user, trigger_request, status)
-      builds_attrs = commit.config_processor.builds_for_stage_and_ref(stage, ref, tag, trigger_request)
+    def initialize(commit)
+      @commit = commit
+    end
+
+    def execute(stage, user, status, trigger_request = nil)
+      builds_attrs = config_processor.builds_for_stage_and_ref(stage, @commit.ref, @commit.tag, trigger_request)
 
       # check when to create next build
       builds_attrs = builds_attrs.select do |build_attrs|
@@ -17,7 +21,8 @@ module Ci
 
       builds_attrs.map do |build_attrs|
         # don't create the same build twice
-        unless commit.builds.find_by(ref: ref, tag: tag, trigger_request: trigger_request, name: build_attrs[:name])
+        unless @commit.builds.find_by(ref: @commit.ref, tag: @commit.tag,
+                                      trigger_request: trigger_request, name: build_attrs[:name])
           build_attrs.slice!(:name,
                              :commands,
                              :tag_list,
@@ -26,17 +31,21 @@ module Ci
                              :stage,
                              :stage_idx)
 
-          build_attrs.merge!(ref: ref,
-                             tag: tag,
+          build_attrs.merge!(ref: @commit.ref,
+                             tag: @commit.tag,
                              trigger_request: trigger_request,
                              user: user,
-                             project: commit.project)
+                             project: @commit.project)
 
-          build = commit.builds.create!(build_attrs)
-          build.execute_hooks
-          build
+          @commit.builds.create!(build_attrs)
         end
       end
     end
+
+    private
+
+    def config_processor
+      @config_processor ||= @commit.config_processor
+    end
   end
 end
diff --git a/app/services/ci/create_trigger_request_service.rb b/app/services/ci/create_trigger_request_service.rb
index b3dfc707221..993acf11db9 100644
--- a/app/services/ci/create_trigger_request_service.rb
+++ b/app/services/ci/create_trigger_request_service.rb
@@ -7,14 +7,14 @@ module Ci
       # check if ref is tag
       tag = project.repository.find_tag(ref).present?
 
-      ci_commit = project.ensure_ci_commit(commit.sha)
+      ci_commit = project.ci_commits.create(sha: commit.sha, ref: ref, tag: tag)
 
       trigger_request = trigger.trigger_requests.create!(
         variables: variables,
         commit: ci_commit,
       )
 
-      if ci_commit.create_builds(ref, tag, nil, trigger_request)
+      if ci_commit.create_builds(nil, trigger_request)
         trigger_request
       end
     end
diff --git a/app/services/ci/image_for_build_service.rb b/app/services/ci/image_for_build_service.rb
index 50c95ced8a7..3018f27ec05 100644
--- a/app/services/ci/image_for_build_service.rb
+++ b/app/services/ci/image_for_build_service.rb
@@ -3,8 +3,9 @@ module Ci
     def execute(project, opts)
       sha = opts[:sha] || ref_sha(project, opts[:ref])
 
-      commit = project.ci_commits.find_by(sha: sha)
-      image_name = image_for_commit(commit)
+      ci_commits = project.ci_commits.where(sha: sha)
+      ci_commits = ci_commits.where(ref: opts[:ref]) if opts[:ref]
+      image_name = image_for_status(ci_commits.status)
 
       image_path = Rails.root.join('public/ci', image_name)
       OpenStruct.new(path: image_path, name: image_name)
@@ -16,9 +17,9 @@ module Ci
       project.commit(ref).try(:sha) if ref
     end
 
-    def image_for_commit(commit)
-      return 'build-unknown.svg' unless commit
-      'build-' + commit.status + ".svg"
+    def image_for_status(status)
+      status ||= 'unknown'
+      'build-' + status + ".svg"
     end
   end
 end
diff --git a/app/services/create_commit_builds_service.rb b/app/services/create_commit_builds_service.rb
index 69d5c42a877..0d2aa1ff03d 100644
--- a/app/services/create_commit_builds_service.rb
+++ b/app/services/create_commit_builds_service.rb
@@ -2,6 +2,7 @@ class CreateCommitBuildsService
   def execute(project, user, params)
     return false unless project.builds_enabled?
 
+    before_sha = params[:checkout_sha] || params[:before]
     sha = params[:checkout_sha] || params[:after]
     origin_ref = params[:ref]
 
@@ -10,15 +11,16 @@ class CreateCommitBuildsService
     end
 
     ref = Gitlab::Git.ref_name(origin_ref)
+    tag = Gitlab::Git.tag_ref?(origin_ref)
 
     # Skip branch removal
     if sha == Gitlab::Git::BLANK_SHA
       return false
     end
 
-    commit = project.ci_commit(sha)
+    commit = project.ci_commit(sha, ref)
     unless commit
-      commit = project.ci_commits.new(sha: sha)
+      commit = project.ci_commits.new(sha: sha, ref: ref, before_sha: before_sha, tag: tag)
 
       # Skip creating ci_commit when no gitlab-ci.yml is found
       unless commit.ci_yaml_file
@@ -32,10 +34,10 @@ class CreateCommitBuildsService
     # Skip creating builds for commits that have [ci skip]
     unless commit.skip_ci?
       # Create builds for commit
-      tag = Gitlab::Git.tag_ref?(origin_ref)
-      commit.create_builds(ref, tag, user)
+      commit.create_builds(user)
     end
 
+    commit.touch
     commit
   end
 end
diff --git a/app/views/projects/_last_commit.html.haml b/app/views/projects/_last_commit.html.haml
index 386d72e7787..66c30283c7a 100644
--- a/app/views/projects/_last_commit.html.haml
+++ b/app/views/projects/_last_commit.html.haml
@@ -1,9 +1,8 @@
 .project-last-commit
-  - ci_commit = project.ci_commit(commit.sha)
-  - if ci_commit
-    = link_to ci_status_path(ci_commit), class: "ci-status ci-#{ci_commit.status}" do
-      = ci_status_icon(ci_commit)
-      = ci_status_label(ci_commit)
+  - if commit.status
+    = link_to builds_namespace_project_commit_path(commit.project.namespace, commit.project, commit), class: "ci-status ci-#{commit.status}" do
+      = ci_icon_for_status(commit.status)
+      = ci_label_for_status(commit.status)
 
   = link_to commit.short_id, namespace_project_commit_path(project.namespace, project, commit), class: "commit_short_id"
   = link_to_gfm commit.title, namespace_project_commit_path(project.namespace, project, commit), class: "commit-row-message"
diff --git a/app/views/projects/builds/index.html.haml b/app/views/projects/builds/index.html.haml
index aa85f495e39..0406fc21d77 100644
--- a/app/views/projects/builds/index.html.haml
+++ b/app/views/projects/builds/index.html.haml
@@ -58,6 +58,6 @@
               %th Coverage
             %th
 
-        = render @builds, commit_sha: true, stage: true, allow_retry: true, coverage: @project.build_coverage_enabled?
+        = render @builds, commit_sha: true, ref: true, stage: true, allow_retry: true, coverage: @project.build_coverage_enabled?
 
     = paginate @builds, theme: 'gitlab'
diff --git a/app/views/projects/builds/show.html.haml b/app/views/projects/builds/show.html.haml
index 6d4505ebb60..99d72aa7935 100644
--- a/app/views/projects/builds/show.html.haml
+++ b/app/views/projects/builds/show.html.haml
@@ -13,7 +13,7 @@
       = link_to "merge request #{merge_request.to_reference}", merge_request_path(merge_request)
 
   #up-build-trace
-  - builds = @build.commit.matrix_builds(@build)
+  - builds = @build.commit.builds.latest.to_a
   - if builds.size > 1
     %ul.nav-links.no-top.no-bottom
       - builds.each do |build|
diff --git a/app/views/projects/ci/builds/_build.html.haml b/app/views/projects/ci/builds/_build.html.haml
index 2cf9115e4dd..e123eb1cc7a 100644
--- a/app/views/projects/ci/builds/_build.html.haml
+++ b/app/views/projects/ci/builds/_build.html.haml
@@ -19,11 +19,12 @@
     %td
       = link_to build.short_sha, namespace_project_commit_path(build.project.namespace, build.project, build.sha), class: "monospace"
 
-  %td
-    - if build.ref
-      = link_to build.ref, namespace_project_commits_path(build.project.namespace, build.project, build.ref)
-    - else
-      .light none
+  - if defined?(ref) && ref
+    %td
+      - if build.ref
+        = link_to build.ref, namespace_project_commits_path(build.project.namespace, build.project, build.ref)
+      - else
+        .light none
 
   - if defined?(runner) && runner
     %td
@@ -48,6 +49,8 @@
         %span.label.label-info triggered
       - if build.try(:allow_failure)
         %span.label.label-danger allowed to fail
+      - if defined?(retried) && retried
+        %span.label.label-warning retried
 
   %td.duration
     - if build.duration
diff --git a/app/views/projects/commit/_builds.html.haml b/app/views/projects/commit/_builds.html.haml
index 003b7c18d0e..5c9a319edeb 100644
--- a/app/views/projects/commit/_builds.html.haml
+++ b/app/views/projects/commit/_builds.html.haml
@@ -1,67 +1,2 @@
-.gray-content-block.middle-block
-  .pull-right
-    - if can?(current_user, :update_build, @ci_commit.project)
-      - if @ci_commit.builds.latest.failed.any?(&:retryable?)
-        = link_to "Retry failed", retry_builds_namespace_project_commit_path(@ci_commit.project.namespace, @ci_commit.project, @ci_commit.sha), class: 'btn btn-grouped btn-primary', method: :post
-
-      - if @ci_commit.builds.running_or_pending.any?
-        = link_to "Cancel running", cancel_builds_namespace_project_commit_path(@ci_commit.project.namespace, @ci_commit.project, @ci_commit.sha), data: { confirm: 'Are you sure?' }, class: 'btn btn-grouped btn-danger', method: :post
-
-  .oneline
-    = pluralize @statuses.count(:id), "build"
-    - if defined?(link_to_commit) && link_to_commit
-      for commit
-      = link_to @ci_commit.short_sha, namespace_project_commit_path(@ci_commit.project.namespace, @ci_commit.project, @ci_commit.sha), class: "monospace"
-    - if @ci_commit.duration > 0
-      in
-      = time_interval_in_words @ci_commit.duration
-
-- if @ci_commit.yaml_errors.present?
-  .bs-callout.bs-callout-danger
-    %h4 Found errors in your .gitlab-ci.yml:
-    %ul
-      - @ci_commit.yaml_errors.split(",").each do |error|
-        %li= error
-    You can also test your .gitlab-ci.yml in the #{link_to "Lint", ci_lint_path}
-
-- if @ci_commit.project.builds_enabled? && !@ci_commit.ci_yaml_file
-  .bs-callout.bs-callout-warning
-    \.gitlab-ci.yml not found in this commit
-
-.table-holder
-  %table.table.builds
-    %thead
-      %tr
-        %th Status
-        %th Build ID
-        %th Ref
-        %th Stage
-        %th Name
-        %th Duration
-        %th Finished at
-        - if @ci_commit.project.build_coverage_enabled?
-          %th Coverage
-        %th
-    - @ci_commit.refs.each do |ref|
-      - builds = @ci_commit.statuses.for_ref(ref).latest.ordered
-      = render builds, coverage: @ci_commit.project.build_coverage_enabled?, stage: true, allow_retry: true
-
-- if @ci_commit.retried.any?
-  .gray-content-block.second-block
-    Retried builds
-
-  .table-holder
-    %table.table.builds
-      %thead
-        %tr
-          %th Status
-          %th Build ID
-          %th Ref
-          %th Stage
-          %th Name
-          %th Duration
-          %th Finished at
-          - if @ci_commit.project.build_coverage_enabled?
-            %th Coverage
-          %th
-      = render @ci_commit.retried, coverage: @ci_commit.project.build_coverage_enabled?, stage: true
+- @ci_commits.each do |ci_commit|
+  = render "ci_commit", ci_commit: ci_commit
diff --git a/app/views/projects/commit/_ci_commit.html.haml b/app/views/projects/commit/_ci_commit.html.haml
new file mode 100644
index 00000000000..25714e6cb47
--- /dev/null
+++ b/app/views/projects/commit/_ci_commit.html.haml
@@ -0,0 +1,69 @@
+.gray-content-block.middle-block
+  .pull-right
+    - if can?(current_user, :update_build, @project)
+      - if ci_commit.builds.latest.failed.any?(&:retryable?)
+        = link_to "Retry failed", retry_builds_namespace_project_commit_path(@project.namespace, @project, ci_commit.sha), class: 'btn btn-grouped btn-primary', method: :post
+
+      - if ci_commit.builds.running_or_pending.any?
+        = link_to "Cancel running", cancel_builds_namespace_project_commit_path(@project.namespace, @project, ci_commit.sha), data: { confirm: 'Are you sure?' }, class: 'btn btn-grouped btn-danger', method: :post
+
+  .oneline
+    = pluralize ci_commit.statuses.count(:id), "build"
+    - if ci_commit.ref
+      for
+      %span.label.label-info
+        = ci_commit.ref
+    - if defined?(link_to_commit) && link_to_commit
+      for commit
+      = link_to ci_commit.short_sha, namespace_project_commit_path(@project.namespace, @project, ci_commit.sha), class: "monospace"
+    - if ci_commit.duration > 0
+      in
+      = time_interval_in_words ci_commit.duration
+
+- if ci_commit.yaml_errors.present?
+  .bs-callout.bs-callout-danger
+    %h4 Found errors in your .gitlab-ci.yml:
+    %ul
+      - ci_commit.yaml_errors.split(",").each do |error|
+        %li= error
+    You can also test your .gitlab-ci.yml in the #{link_to "Lint", ci_lint_path}
+
+- if @project.builds_enabled? && !ci_commit.ci_yaml_file
+  .bs-callout.bs-callout-warning
+    \.gitlab-ci.yml not found in this commit
+
+.table-holder
+  %table.table.builds
+    %thead
+      %tr
+        %th Status
+        %th Build ID
+        %th Stage
+        %th Name
+        %th Duration
+        %th Finished at
+        - if @project.build_coverage_enabled?
+          %th Coverage
+        %th
+    - builds = ci_commit.statuses.latest.ordered
+    = render builds, coverage: @project.build_coverage_enabled?, stage: true, ref: false, allow_retry: true
+
+- if ci_commit.retried.any?
+  .gray-content-block.second-block
+    Retried builds
+
+  .table-holder
+    %table.table.builds
+      %thead
+        %tr
+          %th Status
+          %th Build ID
+          %th Ref
+          %th Stage
+          %th Name
+          %th Duration
+          %th Finished at
+          - if @project.build_coverage_enabled?
+            %th Coverage
+          %th
+      = render ci_commit.retried, coverage: @project.build_coverage_enabled?, stage: true, ref: false
diff --git a/app/views/projects/commit/_commit_box.html.haml b/app/views/projects/commit/_commit_box.html.haml
index d6c9e54e657..3d7c18a5f58 100644
--- a/app/views/projects/commit/_commit_box.html.haml
+++ b/app/views/projects/commit/_commit_box.html.haml
@@ -43,12 +43,12 @@
   - @commit.parents.each do |parent|
     = link_to parent.short_id, namespace_project_commit_path(@project.namespace, @project, parent), class: "monospace"
 
-- if @ci_commit
+- if @commit.status
   .pull-right
-    = link_to ci_status_path(@ci_commit), class: "ci-status ci-#{@ci_commit.status}" do
-      = ci_status_icon(@ci_commit)
+    = link_to builds_namespace_project_commit_path(@project.namespace, @project, @commit.id), class: "ci-status ci-#{@commit.status}" do
+      = ci_icon_for_status(@commit.status)
       build:
-      = ci_status_label(@ci_commit)
+      = ci_label_for_status(@commit.status)
 
 .commit-info-row.branches
   %i.fa.fa-spinner.fa-spin
diff --git a/app/views/projects/commit/show.html.haml b/app/views/projects/commit/show.html.haml
index e550af7888a..e5e3d696035 100644
--- a/app/views/projects/commit/show.html.haml
+++ b/app/views/projects/commit/show.html.haml
@@ -5,7 +5,7 @@
 
 .prepend-top-default
   = render "commit_box"
-- if @ci_commit
+- if @commit.status
   = render "ci_menu"
 - else
   %div.block-connector
diff --git a/app/views/projects/commits/_commit.html.haml b/app/views/projects/commits/_commit.html.haml
index d71f61466f1..c7d8c9a0d15 100644
--- a/app/views/projects/commits/_commit.html.haml
+++ b/app/views/projects/commits/_commit.html.haml
@@ -4,9 +4,8 @@
   - notes = commit.notes
   - note_count = notes.user.count
 
-- ci_commit = project.ci_commit(commit.sha)
 - cache_key = [project.path_with_namespace, commit.id, current_application_settings, note_count]
-- cache_key.push(ci_commit.status) if ci_commit
+- cache_key.push(commit.status) if commit.status
 
 = cache(cache_key) do
   %li.commit.js-toggle-container{ id: "commit-#{commit.short_id}" }
@@ -17,8 +16,8 @@
           %a.text-expander.js-toggle-button ...
 
       .pull-right
-        - if ci_commit
-          = render_ci_status(ci_commit)
+        - if commit.status
+          = render_ci_status(commit)
         = clipboard_button(clipboard_text: commit.id)
         = link_to commit.short_id, namespace_project_commit_path(project.namespace, project, commit), class: "commit_short_id"
 
diff --git a/app/views/projects/generic_commit_statuses/_generic_commit_status.html.haml b/app/views/projects/generic_commit_statuses/_generic_commit_status.html.haml
index c15386b4883..f21c864e35c 100644
--- a/app/views/projects/generic_commit_statuses/_generic_commit_status.html.haml
+++ b/app/views/projects/generic_commit_statuses/_generic_commit_status.html.haml
@@ -15,12 +15,13 @@
   - if defined?(commit_sha) && commit_sha
     %td
       = link_to generic_commit_status.short_sha, namespace_project_commit_path(generic_commit_status.project.namespace, generic_commit_status.project, generic_commit_status.sha), class: "monospace"
-      
-  %td
-    - if generic_commit_status.ref
-      = link_to generic_commit_status.ref, namespace_project_commits_path(generic_commit_status.project.namespace, generic_commit_status.project, generic_commit_status.ref)
-    - else
-      .light none
+
+  - if defined?(ref) && ref
+    %td
+      - if generic_commit_status.ref
+        = link_to generic_commit_status.ref, namespace_project_commits_path(generic_commit_status.project.namespace, generic_commit_status.project, generic_commit_status.ref)
+      - else
+        .light none
 
   - if defined?(runner) && runner
     %td
diff --git a/app/views/projects/issues/_related_branches.html.haml b/app/views/projects/issues/_related_branches.html.haml
index b10cd03515f..bdfa0c7009e 100644
--- a/app/views/projects/issues/_related_branches.html.haml
+++ b/app/views/projects/issues/_related_branches.html.haml
@@ -5,7 +5,7 @@
     - @related_branches.each do |branch|
       %li
         - sha = @project.repository.find_branch(branch).target
-        - ci_commit = @project.ci_commit(sha) if sha
+        - ci_commit = @project.ci_commit(sha, branch) if sha
         - if ci_commit
           %span.related-branch-ci-status
             = render_ci_status(ci_commit)
diff --git a/app/views/projects/merge_requests/show/_builds.html.haml b/app/views/projects/merge_requests/show/_builds.html.haml
index 307a75d02ca..a116ffe2e15 100644
--- a/app/views/projects/merge_requests/show/_builds.html.haml
+++ b/app/views/projects/merge_requests/show/_builds.html.haml
@@ -1 +1,2 @@
-= render "projects/commit/builds", link_to_commit: true
+= render "projects/commit/ci_commit", ci_commit: @ci_commit, link_to_commit: true
+
diff --git a/app/views/shared/projects/_project.html.haml b/app/views/shared/projects/_project.html.haml
index 53ff8959bc8..ab8b022411d 100644
--- a/app/views/shared/projects/_project.html.haml
+++ b/app/views/shared/projects/_project.html.haml
@@ -6,9 +6,8 @@
 - css_class = '' unless local_assigns[:css_class]
 - show_last_commit_as_description = false unless local_assigns[:show_last_commit_as_description] == true && project.commit
 - css_class += " no-description" if project.description.blank? && !show_last_commit_as_description
-- ci_commit = project.ci_commit(project.commit.sha) if ci && !project.empty_repo? && project.commit
 - cache_key = [project.namespace, project, controller.controller_name, controller.action_name, current_application_settings, 'v2.3']
-- cache_key.push(ci_commit.status) if ci_commit
+- cache_key.push(project.commit.status) if project.commit.try(:status)
 
 %li.project-row{ class: css_class }
   = cache(cache_key) do
@@ -16,9 +15,9 @@
       - if project.main_language
         %span
           = project.main_language
-      - if ci_commit
+      - if project.commit.try(:status)
         %span
-          = render_ci_status(ci_commit)
+          = render_ci_status(project.commit)
       - if forks
         %span
           = icon('code-fork')