8 files changed, 241 insertions, 261 deletions

diff --git a/app/models/ci/artifact_blob.rb b/app/models/ci/artifact_blob.rb
new file mode 100644
index 00000000000..b35febc9ac5
--- /dev/null
+++ b/app/models/ci/artifact_blob.rb
@@ -0,0 +1,35 @@
+module Ci
+  class ArtifactBlob
+    include BlobLike
+
+    attr_reader :entry
+
+    def initialize(entry)
+      @entry = entry
+    end
+
+    delegate :name, :path, to: :entry
+
+    def id
+      Digest::SHA1.hexdigest(path)
+    end
+
+    def size
+      entry.metadata[:size]
+    end
+
+    def data
+      "Build artifact #{path}"
+    end
+
+    def mode
+      entry.metadata[:mode]
+    end
+
+    def external_storage
+      :build_artifact
+    end
+
+    alias_method :external_size, :size
+  end
+end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index dbe4a2bf43f..1581ba9e55d 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -103,27 +103,17 @@ module Ci
     end
 
     def playable?
-      project.builds_enabled? && has_commands? &&
-        action? && manual?
+      action? && manual?
     end
 
     def action?
       self.when == 'manual'
     end
 
-    def has_commands?
-      commands.present?
-    end
-
     def play(current_user)
-      # Try to queue a current build
-      if self.enqueue
-        self.update(user: current_user)
-        self
-      else
-        # Otherwise we need to create a duplicate
-        Ci::Build.retry(self, current_user)
-      end
+      Ci::PlayBuildService
+        .new(project, current_user)
+        .execute(self)
     end
 
     def cancelable?
@@ -131,12 +121,11 @@ module Ci
     end
 
     def retryable?
-      project.builds_enabled? && has_commands? &&
-        (success? || failed? || canceled?)
+      success? || failed? || canceled?
     end
 
-    def retried?
-      !self.pipeline.statuses.latest.include?(self)
+    def latest?
+      !retried?
     end
 
     def expanded_environment_name
@@ -171,19 +160,6 @@ module Ci
       latest_builds.where('stage_idx < ?', stage_idx)
     end
 
-    def trace_html(**args)
-      trace_with_state(**args)[:html] || ''
-    end
-
-    def trace_with_state(state: nil, last_lines: nil)
-      trace_ansi = trace(last_lines: last_lines)
-      if trace_ansi.present?
-        Ci::Ansi2html.convert(trace_ansi, state)
-      else
-        {}
-      end
-    end
-
     def timeout
       project.build_timeout
     end
@@ -244,136 +220,35 @@ module Ci
     end
 
     def update_coverage
-      coverage = extract_coverage(trace, coverage_regex)
+      coverage = trace.extract_coverage(coverage_regex)
       update_attributes(coverage: coverage) if coverage.present?
     end
 
-    def extract_coverage(text, regex)
-      return unless regex
-
-      matches = text.scan(Regexp.new(regex)).last
-      matches = matches.last if matches.is_a?(Array)
-      coverage = matches.gsub(/\d+(\.\d+)?/).first
-
-      if coverage.present?
-        coverage.to_f
-      end
-    rescue
-      # if bad regex or something goes wrong we dont want to interrupt transition
-      # so we just silentrly ignore error for now
-    end
-
-    def has_trace_file?
-      File.exist?(path_to_trace) || has_old_trace_file?
+    def trace
+      Gitlab::Ci::Trace.new(self)
     end
 
     def has_trace?
-      raw_trace.present?
-    end
-
-    def raw_trace(last_lines: nil)
-      if File.exist?(trace_file_path)
-        Gitlab::Ci::TraceReader.new(trace_file_path).
-          read(last_lines: last_lines)
-      else
-        # backward compatibility
-        read_attribute :trace
-      end
-    end
-
-    ##
-    # Deprecated
-    #
-    # This is a hotfix for CI build data integrity, see #4246
-    def has_old_trace_file?
-      project.ci_id && File.exist?(old_path_to_trace)
-    end
-
-    def trace(last_lines: nil)
-      hide_secrets(raw_trace(last_lines: last_lines))
-    end
-
-    def trace_length
-      if raw_trace
-        raw_trace.bytesize
-      else
-        0
-      end
+      trace.exist?
     end
 
-    def trace=(trace)
-      recreate_trace_dir
-      trace = hide_secrets(trace)
-      File.write(path_to_trace, trace)
+    def trace=(data)
+      raise NotImplementedError
     end
 
-    def recreate_trace_dir
-      unless Dir.exist?(dir_to_trace)
-        FileUtils.mkdir_p(dir_to_trace)
-      end
+    def old_trace
+      read_attribute(:trace)
     end
-    private :recreate_trace_dir
-
-    def append_trace(trace_part, offset)
-      recreate_trace_dir
-      touch if needs_touch?
-
-      trace_part = hide_secrets(trace_part)
 
-      File.truncate(path_to_trace, offset) if File.exist?(path_to_trace)
-      File.open(path_to_trace, 'ab') do |f|
-        f.write(trace_part)
-      end
+    def erase_old_trace!
+      write_attribute(:trace, nil)
+      save
     end
 
     def needs_touch?
       Time.now - updated_at > 15.minutes.to_i
     end
 
-    def trace_file_path
-      if has_old_trace_file?
-        old_path_to_trace
-      else
-        path_to_trace
-      end
-    end
-
-    def dir_to_trace
-      File.join(
-        Settings.gitlab_ci.builds_path,
-        created_at.utc.strftime("%Y_%m"),
-        project.id.to_s
-      )
-    end
-
-    def path_to_trace
-      "#{dir_to_trace}/#{id}.log"
-    end
-
-    ##
-    # Deprecated
-    #
-    # This is a hotfix for CI build data integrity, see #4246
-    # Should be removed in 8.4, after CI files migration has been done.
-    #
-    def old_dir_to_trace
-      File.join(
-        Settings.gitlab_ci.builds_path,
-        created_at.utc.strftime("%Y_%m"),
-        project.ci_id.to_s
-      )
-    end
-
-    ##
-    # Deprecated
-    #
-    # This is a hotfix for CI build data integrity, see #4246
-    # Should be removed in 8.4, after CI files migration has been done.
-    #
-    def old_path_to_trace
-      "#{old_dir_to_trace}/#{id}.log"
-    end
-
     ##
     # Deprecated
     #
@@ -425,8 +300,8 @@ module Ci
     def execute_hooks
       return unless project
       build_data = Gitlab::DataBuilder::Build.build(self)
-      project.execute_hooks(build_data.dup, :build_hooks)
-      project.execute_services(build_data.dup, :build_hooks)
+      project.execute_hooks(build_data.dup, :job_hooks)
+      project.execute_services(build_data.dup, :job_hooks)
       PagesService.new(build_data).execute
       project.running_or_pending_build_count(force: true)
     end
@@ -540,6 +415,8 @@ module Ci
     end
 
     def dependencies
+      return [] if empty_dependencies?
+
       depended_jobs = depends_on_builds
 
       return depended_jobs unless options[:dependencies].present?
@@ -549,6 +426,19 @@ module Ci
       end
     end
 
+    def empty_dependencies?
+      options[:dependencies]&.empty?
+    end
+
+    def hide_secrets(trace)
+      return unless trace
+
+      trace = trace.dup
+      Ci::MaskSecret.mask!(trace, project.runners_token) if project
+      Ci::MaskSecret.mask!(trace, token)
+      trace
+    end
+
     private
 
     def update_artifacts_size
@@ -560,7 +450,7 @@ module Ci
     end
 
     def erase_trace!
-      self.trace = nil
+      trace.erase!
     end
 
     def update_erased!(user = nil)
@@ -622,15 +512,6 @@ module Ci
       pipeline.config_processor.build_attributes(name)
     end
 
-    def hide_secrets(trace)
-      return unless trace
-
-      trace = trace.dup
-      Ci::MaskSecret.mask!(trace, project.runners_token) if project
-      Ci::MaskSecret.mask!(trace, token)
-      trace
-    end
-
     def update_project_statistics
       return unless project
 
diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb
new file mode 100644
index 00000000000..87898b086c6
--- /dev/null
+++ b/app/models/ci/group.rb
@@ -0,0 +1,40 @@
+module Ci
+  ##
+  # This domain model is a representation of a group of jobs that are related
+  # to each other, like `rspec 0 1`, `rspec 0 2`.
+  #
+  # It is not persisted in the database.
+  #
+  class Group
+    include StaticModel
+
+    attr_reader :stage, :name, :jobs
+
+    delegate :size, to: :jobs
+
+    def initialize(stage, name:, jobs:)
+      @stage = stage
+      @name = name
+      @jobs = jobs
+    end
+
+    def status
+      @status ||= commit_statuses.status
+    end
+
+    def detailed_status(current_user)
+      if jobs.one?
+        jobs.first.detailed_status(current_user)
+      else
+        Gitlab::Ci::Status::Group::Factory
+          .new(self, current_user).fabricate!
+      end
+    end
+
+    private
+
+    def commit_statuses
+      @commit_statuses ||= CommitStatus.where(id: jobs.map(&:id))
+    end
+  end
+end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index e079072a23f..fa1312154ca 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -4,14 +4,30 @@ module Ci
     include HasStatus
     include Importable
     include AfterCommitQueue
+    include Presentable
 
     belongs_to :project
     belongs_to :user
+    belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline'
+    belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule'
+
+    has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
+    has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
 
     has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id
     has_many :builds, foreign_key: :commit_id
     has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id
 
+    # Merge requests for which the current pipeline is running against
+    # the merge request's latest commit.
+    has_many :merge_requests, foreign_key: "head_pipeline_id"
+
+    has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build'
+    has_many :retryable_builds, -> { latest.failed_or_canceled }, foreign_key: :commit_id, class_name: 'Ci::Build'
+    has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus'
+    has_many :manual_actions, -> { latest.manual_actions }, foreign_key: :commit_id, class_name: 'Ci::Build'
+    has_many :artifacts, -> { latest.with_artifacts_not_expired }, foreign_key: :commit_id, class_name: 'Ci::Build'
+
     delegate :id, to: :project, prefix: true
 
     validates :sha, presence: { unless: :importing? }
@@ -20,7 +36,6 @@ module Ci
     validate :valid_commit_sha, unless: :importing?
 
     after_create :keep_around_commits, unless: :importing?
-    after_create :refresh_build_status_cache
 
     state_machine :status, initial: :created do
       event :enqueue do
@@ -65,23 +80,32 @@ module Ci
         pipeline.update_duration
       end
 
+      before_transition any => [:manual] do |pipeline|
+        pipeline.update_duration
+      end
+
+      before_transition canceled: any - [:canceled] do |pipeline|
+        pipeline.auto_canceled_by = nil
+      end
+
       after_transition [:created, :pending] => :running do |pipeline|
-        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(id) }
+        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
       end
 
       after_transition any => [:success] do |pipeline|
-        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(id) }
+        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
       end
 
       after_transition [:created, :pending, :running] => :success do |pipeline|
-        pipeline.run_after_commit { PipelineSuccessWorker.perform_async(id) }
+        pipeline.run_after_commit { PipelineSuccessWorker.perform_async(pipeline.id) }
       end
 
       after_transition do |pipeline, transition|
         next if transition.loopback?
 
         pipeline.run_after_commit do
-          PipelineHooksWorker.perform_async(id)
+          PipelineHooksWorker.perform_async(pipeline.id)
+          ExpirePipelineCacheWorker.perform_async(pipeline.id)
         end
       end
 
@@ -247,11 +271,37 @@ module Ci
       statuses_with(status: HasStatus::CANCELABLE_STATUSES).any?
     end
 
+    def stuck?
+      pending_builds.any?(&:stuck?)
+    end
+
+    def retryable?
+      retryable_builds.any?
+    end
+
+    def cancelable?
+      cancelable_statuses.any?
+    end
+
+    def auto_canceled?
+      canceled? && auto_canceled_by_id?
+    end
+
     def cancel_running
-      Gitlab::OptimisticLocking.retry_lock(
-        statuses.cancelable) do |cancelable|
-          cancelable.find_each(&:cancel)
+      Gitlab::OptimisticLocking.retry_lock(cancelable_statuses) do |cancelable|
+        cancelable.find_each do |job|
+          yield(job) if block_given?
+          job.cancel
         end
+      end
+    end
+
+    def auto_cancel_running(pipeline)
+      update(auto_canceled_by: pipeline)
+
+      cancel_running do |job|
+        job.auto_canceled_by = pipeline
+      end
     end
 
     def retry_failed(current_user)
@@ -359,7 +409,6 @@ module Ci
         when 'manual' then block
         end
       end
-      refresh_build_status_cache
     end
 
     def predefined_variables
@@ -387,12 +436,9 @@ module Ci
       project.execute_services(data, :pipeline_hooks)
     end
 
-    # Merge requests for which the current pipeline is running against
-    # the merge request's latest commit.
-    def merge_requests
-      @merge_requests ||= project.merge_requests
-        .where(source_branch: self.ref)
-        .select { |merge_request| merge_request.head_pipeline.try(:id) == self.id }
+    # All the merge requests for which the current pipeline runs/ran against
+    def all_merge_requests
+      @all_merge_requests ||= project.merge_requests.where(source_branch: ref)
     end
 
     def detailed_status(current_user)
@@ -401,10 +447,6 @@ module Ci
         .fabricate!
     end
 
-    def refresh_build_status_cache
-      Ci::PipelineStatus.new(project, sha: sha, status: status).store_in_cache_if_needed
-    end
-
     private
 
     def pipeline_data
diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb
new file mode 100644
index 00000000000..cf6e53c4ca4
--- /dev/null
+++ b/app/models/ci/pipeline_schedule.rb
@@ -0,0 +1,60 @@
+module Ci
+  class PipelineSchedule < ActiveRecord::Base
+    extend Ci::Model
+    include Importable
+
+    acts_as_paranoid
+
+    belongs_to :project
+    belongs_to :owner, class_name: 'User'
+    has_one :last_pipeline, -> { order(id: :desc) }, class_name: 'Ci::Pipeline'
+    has_many :pipelines
+
+    validates :cron, unless: :importing_or_inactive?, cron: true, presence: { unless: :importing_or_inactive? }
+    validates :cron_timezone, cron_timezone: true, presence: { unless: :importing_or_inactive? }
+    validates :ref, presence: { unless: :importing_or_inactive? }
+    validates :description, presence: true
+
+    before_save :set_next_run_at
+
+    scope :active, -> { where(active: true) }
+    scope :inactive, -> { where(active: false) }
+
+    def owned_by?(current_user)
+      owner == current_user
+    end
+
+    def inactive?
+      !active?
+    end
+
+    def deactivate!
+      update_attribute(:active, false)
+    end
+
+    def importing_or_inactive?
+      importing? || inactive?
+    end
+
+    def runnable_by_owner?
+      Ability.allowed?(owner, :create_pipeline, project)
+    end
+
+    def set_next_run_at
+      self.next_run_at = Gitlab::Ci::CronParser.new(cron, cron_timezone).next_time_from(Time.now)
+    end
+
+    def schedule_next_run!
+      save! # with set_next_run_at
+    rescue ActiveRecord::RecordInvalid
+      update_attribute(:next_run_at, nil) # update without validation
+    end
+
+    def real_next_run(
+        worker_cron: Settings.cron_jobs['pipeline_schedule_worker']['cron'],
+        worker_time_zone: Time.zone.name)
+      Gitlab::Ci::CronParser.new(worker_cron, worker_time_zone)
+                            .next_time_from(next_run_at)
+    end
+  end
+end
diff --git a/app/models/ci/pipeline_status.rb b/app/models/ci/pipeline_status.rb
deleted file mode 100644
index 048047d0e34..00000000000
--- a/app/models/ci/pipeline_status.rb
+++ /dev/null
@@ -1,86 +0,0 @@
-# This class is not backed by a table in the main database.
-# It loads the latest Pipeline for the HEAD of a repository, and caches that
-# in Redis.
-module Ci
-  class PipelineStatus
-    attr_accessor :sha, :status, :project, :loaded
-
-    delegate :commit, to: :project
-
-    def self.load_for_project(project)
-      new(project).tap do |status|
-        status.load_status
-      end
-    end
-
-    def initialize(project, sha: nil, status: nil)
-      @project = project
-      @sha = sha
-      @status = status
-    end
-
-    def has_status?
-      loaded? && sha.present? && status.present?
-    end
-
-    def load_status
-      return if loaded?
-
-      if has_cache?
-        load_from_cache
-      else
-        load_from_commit
-        store_in_cache
-      end
-
-      self.loaded = true
-    end
-
-    def load_from_commit
-      return unless commit
-
-      self.sha = commit.sha
-      self.status = commit.status
-    end
-
-    # We only cache the status for the HEAD commit of a project
-    # This status is rendered in project lists
-    def store_in_cache_if_needed
-      return unless sha
-      return delete_from_cache unless commit
-      store_in_cache if commit.sha == self.sha
-    end
-
-    def load_from_cache
-      Gitlab::Redis.with do |redis|
-        self.sha, self.status = redis.hmget(cache_key, :sha, :status)
-      end
-    end
-
-    def store_in_cache
-      Gitlab::Redis.with do |redis|
-        redis.mapped_hmset(cache_key, { sha: sha, status: status })
-      end
-    end
-
-    def delete_from_cache
-      Gitlab::Redis.with do |redis|
-        redis.del(cache_key)
-      end
-    end
-
-    def has_cache?
-      Gitlab::Redis.with do |redis|
-        redis.exists(cache_key)
-      end
-    end
-
-    def loaded?
-      self.loaded
-    end
-
-    def cache_key
-      "projects/#{project.id}/build_status"
-    end
-  end
-end
diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb
index e7d6b17d445..9bda3186c30 100644
--- a/app/models/ci/stage.rb
+++ b/app/models/ci/stage.rb
@@ -15,6 +15,14 @@ module Ci
       @warnings = warnings
     end
 
+    def groups
+      @groups ||= statuses.ordered.latest
+        .sort_by(&:sortable_name).group_by(&:group_name)
+        .map do |group_name, grouped_statuses|
+          Ci::Group.new(self, name: group_name, jobs: grouped_statuses)
+        end
+    end
+
     def to_param
       name
     end
diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb
index cba1d81a861..6df41a3f301 100644
--- a/app/models/ci/trigger.rb
+++ b/app/models/ci/trigger.rb
@@ -7,7 +7,7 @@ module Ci
     belongs_to :project
     belongs_to :owner, class_name: "User"
 
-    has_many :trigger_requests, dependent: :destroy
+    has_many :trigger_requests
 
     validates :token, presence: true, uniqueness: true