Merge branch 'ci-commit-as-pipeline' into 'master'

Ci::Commit becomes a Pipeline object

1. Ci::Commit receives context: ref, :tag.
1. One Ci::Commit describes a one Pipeline
1. Pipeline is created from `.gitlab-ci.yml`
1. Pipeline is a ordered group of builds
1. We test MR against Pipeline
1. Pipelines have a separate view (https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3703)
1. Pipeline can be triggered from UI (https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3703)
1. Later we change `Trigger -> TriggerRequest -> Build` to `Trigger -> Pipeline` (future)
1. We add a Pipeline Hook that will be triggered on Pipeline status change  (future)
1. We extend notifications to use `Pipeline Hook` to send summary on pipeline changes (future)

After merging that I'll prepare a separate MR that will unify naming, database columns, table names:
```
Ci::Commit -> Pipeline
Ci::Build -> Build
CommitStatus -> Job
GenericCommitStatus -> ExternalJob

ci_commits -> pipelines
ci_builds -> jobs
```

This MR implements first 5 points.

This is made to solve this issue https://gitlab.com/gitlab-org/gitlab-ce/issues/14149.

See merge request !3653

7 files changed, 173 insertions, 145 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index d42a65620ff..553cd447971 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -37,8 +37,6 @@
 
 module Ci
   class Build < CommitStatus
-    LAZY_ATTRIBUTES = ['trace']
-
     belongs_to :runner, class_name: 'Ci::Runner'
     belongs_to :trigger_request, class_name: 'Ci::TriggerRequest'
     belongs_to :erased_by, class_name: 'User'
@@ -50,25 +48,17 @@ module Ci
 
     scope :unstarted, ->() { where(runner_id: nil) }
     scope :ignore_failures, ->() { where(allow_failure: false) }
-    scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) }
 
     mount_uploader :artifacts_file, ArtifactUploader
     mount_uploader :artifacts_metadata, ArtifactUploader
 
     acts_as_taggable
 
-    # To prevent db load megabytes of data from trace
-    default_scope -> { select(Ci::Build.columns_without_lazy) }
-
     before_destroy { project }
 
-    class << self
-      def columns_without_lazy
-        (column_names - LAZY_ATTRIBUTES).map do |column_name|
-          "#{table_name}.#{column_name}"
-        end
-      end
+    after_create :execute_hooks
 
+    class << self
       def last_month
         where('created_at > ?', Date.today - 1.month)
       end
@@ -126,12 +116,16 @@ module Ci
     end
 
     def retried?
-      !self.commit.latest_statuses_for_ref(self.ref).include?(self)
+      !self.commit.statuses.latest.include?(self)
+    end
+
+    def retry
+      Ci::Build.retry(self)
     end
 
     def depends_on_builds
       # Get builds of the same type
-      latest_builds = self.commit.builds.similar(self).latest
+      latest_builds = self.commit.builds.latest
 
       # Return builds from previous stages
       latest_builds.where('stage_idx < ?', stage_idx)
diff --git a/app/models/ci/commit.rb b/app/models/ci/commit.rb
index f4cf7034b14..f2667e5476b 100644
--- a/app/models/ci/commit.rb
+++ b/app/models/ci/commit.rb
@@ -19,21 +19,28 @@
 module Ci
   class Commit < ActiveRecord::Base
     extend Ci::Model
+    include Statuseable
 
     belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id
     has_many :statuses, class_name: 'CommitStatus'
     has_many :builds, class_name: 'Ci::Build'
     has_many :trigger_requests, dependent: :destroy, class_name: 'Ci::TriggerRequest'
 
+    delegate :stages, to: :statuses
+
     validates_presence_of :sha
+    validates_presence_of :status
     validate :valid_commit_sha
 
+    # Invalidate object and save if when touched
+    after_touch :update_state
+
     def self.truncate_sha(sha)
       sha[0...8]
     end
 
-    def to_param
-      sha
+    def self.stages
+      CommitStatus.where(commit: all).stages
     end
 
     def project_id
@@ -68,15 +75,20 @@ module Ci
       nil
     end
 
-    def stage
-      running_or_pending = statuses.latest.running_or_pending.ordered
-      running_or_pending.first.try(:stage)
+    def branch?
+      !tag?
+    end
+
+    def retryable?
+      builds.latest.any? do |build|
+        build.failed? && build.retryable?
+      end
     end
 
-    def create_builds(ref, tag, user, trigger_request = nil)
+    def create_builds(user, trigger_request = nil)
       return unless config_processor
       config_processor.stages.any? do |stage|
-        CreateBuildsService.new.execute(self, stage, ref, tag, user, trigger_request, 'success').present?
+        CreateBuildsService.new(self).execute(stage, user, 'success', trigger_request).present?
       end
     end
 
@@ -84,7 +96,7 @@ module Ci
       return unless config_processor
 
       # don't create other builds if this one is retried
-      latest_builds = builds.similar(build).latest
+      latest_builds = builds.latest
       return unless latest_builds.exists?(build.id)
 
       # get list of stages after this build
@@ -92,88 +104,21 @@ module Ci
       next_stages.delete(build.stage)
 
       # get status for all prior builds
-      prior_builds = latest_builds.reject { |other_build| next_stages.include?(other_build.stage) }
-      status = Ci::Status.get_status(prior_builds)
+      prior_builds = latest_builds.where.not(stage: next_stages)
+      prior_status = prior_builds.status
 
       # create builds for next stages based
       next_stages.any? do |stage|
-        CreateBuildsService.new.execute(self, stage, build.ref, build.tag, build.user, build.trigger_request, status).present?
+        CreateBuildsService.new(self).execute(stage, build.user, prior_status, build.trigger_request).present?
       end
     end
 
-    def refs
-      statuses.order(:ref).pluck(:ref).uniq
-    end
-
-    def latest_statuses
-      @latest_statuses ||= statuses.latest.to_a
-    end
-
-    def latest_statuses_for_ref(ref)
-      latest_statuses.select { |status| status.ref == ref }
-    end
-
-    def matrix_builds(build = nil)
-      matrix_builds = builds.latest.ordered
-      matrix_builds = matrix_builds.similar(build) if build
-      matrix_builds.to_a
-    end
-
     def retried
       @retried ||= (statuses.order(id: :desc) - statuses.latest)
     end
 
-    def status
-      if yaml_errors.present?
-        return 'failed'
-      end
-
-      @status ||= Ci::Status.get_status(latest_statuses)
-    end
-
-    def pending?
-      status == 'pending'
-    end
-
-    def running?
-      status == 'running'
-    end
-
-    def success?
-      status == 'success'
-    end
-
-    def failed?
-      status == 'failed'
-    end
-
-    def canceled?
-      status == 'canceled'
-    end
-
-    def active?
-      running? || pending?
-    end
-
-    def complete?
-      canceled? || success? || failed?
-    end
-
-    def duration
-      duration_array = statuses.map(&:duration).compact
-      duration_array.reduce(:+).to_i
-    end
-
-    def started_at
-      @started_at ||= statuses.order('started_at ASC').first.try(:started_at)
-    end
-
-    def finished_at
-      @finished_at ||= statuses.order('finished_at DESC').first.try(:finished_at)
-    end
-
     def coverage
-      coverage_array = latest_statuses.map(&:coverage).compact
+      coverage_array = statuses.latest.map(&:coverage).compact
       if coverage_array.size >= 1
         '%.2f' % (coverage_array.reduce(:+) / coverage_array.size)
       end
@@ -181,23 +126,29 @@ module Ci
 
     def config_processor
       return nil unless ci_yaml_file
-      @config_processor ||= Ci::GitlabCiYamlProcessor.new(ci_yaml_file, project.path_with_namespace)
-    rescue Ci::GitlabCiYamlProcessor::ValidationError, Psych::SyntaxError => e
-      save_yaml_error(e.message)
-      nil
-    rescue
-      save_yaml_error("Undefined error")
-      nil
+      return @config_processor if defined?(@config_processor)
+
+      @config_processor ||= begin
+        Ci::GitlabCiYamlProcessor.new(ci_yaml_file, project.path_with_namespace)
+      rescue Ci::GitlabCiYamlProcessor::ValidationError, Psych::SyntaxError => e
+        save_yaml_error(e.message)
+        nil
+      rescue
+        save_yaml_error("Undefined error")
+        nil
+      end
     end
 
     def ci_yaml_file
+      return @ci_yaml_file if defined?(@ci_yaml_file)
+
       @ci_yaml_file ||= begin
         blob = project.repository.blob_at(sha, '.gitlab-ci.yml')
         blob.load_all_data!(project.repository)
         blob.data
+      rescue
+        nil
       end
-    rescue
-      nil
     end
 
     def skip_ci?
@@ -206,10 +157,23 @@ module Ci
 
     private
 
+    def update_state
+      statuses.reload
+      self.status = if yaml_errors.blank?
+                      statuses.latest.status || 'skipped'
+                    else
+                      'failed'
+                    end
+      self.started_at = statuses.started_at
+      self.finished_at = statuses.finished_at
+      self.duration = statuses.latest.duration
+      save
+    end
+
     def save_yaml_error(error)
       return if self.yaml_errors?
       self.yaml_errors = error
-      save
+      update_state
     end
   end
 end
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 6bb018b086f..562c3ed15b2 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -207,12 +207,13 @@ class Commit
     @raw.short_id(7)
   end
 
-  def ci_commit
-    project.ci_commit(sha)
+  def ci_commits
+    @ci_commits ||= project.ci_commits.where(sha: sha)
   end
 
   def status
-    ci_commit.try(:status) || :not_found
+    return @status if defined?(@status)
+    @status ||= ci_commits.status
   end
 
   def revert_branch_name
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 3377a85a55a..aa56314aa16 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -33,30 +33,23 @@
 #
 
 class CommitStatus < ActiveRecord::Base
+  include Statuseable
+
   self.table_name = 'ci_builds'
 
   belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id
-  belongs_to :commit, class_name: 'Ci::Commit'
+  belongs_to :commit, class_name: 'Ci::Commit', touch: true
   belongs_to :user
 
   validates :commit, presence: true
-  validates :status, inclusion: { in: %w(pending running failed success canceled) }
 
   validates_presence_of :name
 
   alias_attribute :author, :user
 
-  scope :running, -> { where(status: 'running') }
-  scope :pending, -> { where(status: 'pending') }
-  scope :success, -> { where(status: 'success') }
-  scope :failed, -> { where(status: 'failed')  }
-  scope :running_or_pending, -> { where(status: [:running, :pending]) }
-  scope :finished, -> { where(status: [:success, :failed, :canceled]) }
-  scope :latest, -> { where(id: unscope(:select).select('max(id)').group(:name, :ref)) }
+  scope :latest, -> { where(id: unscope(:select).select('max(id)').group(:name, :commit_id)) }
   scope :ordered, -> { order(:ref, :stage_idx, :name) }
-  scope :for_ref, ->(ref) { where(ref: ref) }
-
-  AVAILABLE_STATUSES = ['pending', 'running', 'success', 'failed', 'canceled']
+  scope :ignored, -> { where(allow_failure: true, status: [:failed, :canceled]) }
 
   state_machine :status, initial: :pending do
     event :run do
@@ -86,31 +79,24 @@ class CommitStatus < ActiveRecord::Base
     after_transition [:pending, :running] => :success do |commit_status|
       MergeRequests::MergeWhenBuildSucceedsService.new(commit_status.commit.project, nil).trigger(commit_status)
     end
-
-    state :pending, value: 'pending'
-    state :running, value: 'running'
-    state :failed, value: 'failed'
-    state :success, value: 'success'
-    state :canceled, value: 'canceled'
   end
 
-  delegate :sha, :short_sha, to: :commit, prefix: false
+  delegate :sha, :short_sha, to: :commit
 
-  # TODO: this should be removed with all references
   def before_sha
-    Gitlab::Git::BLANK_SHA
+    commit.before_sha || Gitlab::Git::BLANK_SHA
   end
 
-  def started?
-    !pending? && !canceled? && started_at
+  def self.stages
+    order_by = 'max(stage_idx)'
+    group('stage').order(order_by).pluck(:stage, order_by).map(&:first).compact
   end
 
-  def active?
-    running? || pending?
-  end
-
-  def complete?
-    canceled? || success? || failed?
+  def self.stages_status
+    all.stages.inject({}) do |h, stage|
+      h[stage] = all.where(stage: stage).status
+      h
+    end
   end
 
   def ignored?
@@ -118,11 +104,13 @@ class CommitStatus < ActiveRecord::Base
   end
 
   def duration
-    if started_at && finished_at
-      finished_at - started_at
-    elsif started_at
-      Time.now - started_at
-    end
+    duration =
+      if started_at && finished_at
+        finished_at - started_at
+      elsif started_at
+        Time.now - started_at
+      end
+    duration
   end
 
   def stuck?
diff --git a/app/models/concerns/statuseable.rb b/app/models/concerns/statuseable.rb
new file mode 100644
index 00000000000..8a293b7b76e
--- /dev/null
+++ b/app/models/concerns/statuseable.rb
@@ -0,0 +1,81 @@
+module Statuseable
+  extend ActiveSupport::Concern
+
+  AVAILABLE_STATUSES = %w(pending running success failed canceled skipped)
+
+  class_methods do
+    def status_sql
+      builds = all.select('count(*)').to_sql
+      success = all.success.select('count(*)').to_sql
+      ignored = all.ignored.select('count(*)').to_sql if all.respond_to?(:ignored)
+      ignored ||= '0'
+      pending = all.pending.select('count(*)').to_sql
+      running = all.running.select('count(*)').to_sql
+      canceled = all.canceled.select('count(*)').to_sql
+      skipped = all.skipped.select('count(*)').to_sql
+
+      deduce_status = "(CASE
+        WHEN (#{builds})=0 THEN NULL
+        WHEN (#{builds})=(#{success})+(#{ignored}) THEN 'success'
+        WHEN (#{builds})=(#{pending}) THEN 'pending'
+        WHEN (#{builds})=(#{canceled}) THEN 'canceled'
+        WHEN (#{builds})=(#{skipped}) THEN 'skipped'
+        WHEN (#{running})+(#{pending})>0 THEN 'running'
+        ELSE 'failed'
+      END)"
+
+      deduce_status
+    end
+
+    def status
+      all.pluck(self.status_sql).first
+    end
+
+    def duration
+      duration_array = all.map(&:duration).compact
+      duration_array.reduce(:+)
+    end
+
+    def started_at
+      all.minimum(:started_at)
+    end
+
+    def finished_at
+      all.maximum(:finished_at)
+    end
+  end
+
+  included do
+    validates :status, inclusion: { in: AVAILABLE_STATUSES }
+
+    state_machine :status, initial: :pending do
+      state :pending, value: 'pending'
+      state :running, value: 'running'
+      state :failed, value: 'failed'
+      state :success, value: 'success'
+      state :canceled, value: 'canceled'
+      state :skipped, value: 'skipped'
+    end
+
+    scope :running, -> { where(status: 'running') }
+    scope :pending, -> { where(status: 'pending') }
+    scope :success, -> { where(status: 'success') }
+    scope :failed, -> { where(status: 'failed')  }
+    scope :canceled, -> { where(status: 'canceled')  }
+    scope :skipped, -> { where(status: 'skipped')  }
+    scope :running_or_pending, -> { where(status: [:running, :pending]) }
+    scope :finished, -> { where(status: [:success, :failed, :canceled]) }
+  end
+
+  def started?
+    !pending? && !canceled? && started_at
+  end
+
+  def active?
+    running? || pending?
+  end
+
+  def complete?
+    canceled? || success? || failed?
+  end
+end
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index dbecc48485c..d00919c3b0c 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -586,7 +586,7 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def ci_commit
-    @ci_commit ||= source_project.ci_commit(last_commit.id) if last_commit && source_project
+    @ci_commit ||= source_project.ci_commit(last_commit.id, source_branch) if last_commit && source_project
   end
 
   def diff_refs
diff --git a/app/models/project.rb b/app/models/project.rb
index 8f0272d2ce0..7aa21b19e67 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -963,12 +963,12 @@ class Project < ActiveRecord::Base
     !namespace.share_with_group_lock
   end
 
-  def ci_commit(sha)
-    ci_commits.find_by(sha: sha)
+  def ci_commit(sha, ref)
+    ci_commits.order(id: :desc).find_by(sha: sha, ref: ref)
   end
 
-  def ensure_ci_commit(sha)
-    ci_commit(sha) || ci_commits.create(sha: sha)
+  def ensure_ci_commit(sha, ref)
+    ci_commit(sha, ref) || ci_commits.create(sha: sha, ref: ref)
   end
 
   def enable_ci