5 files changed, 115 insertions, 250 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index ad0be70c32a..b426c27afbb 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -103,18 +103,13 @@ module Ci
     end
 
     def playable?
-      project.builds_enabled? && has_commands? &&
-        action? && manual?
+      action? && manual?
     end
 
     def action?
       self.when == 'manual'
     end
 
-    def has_commands?
-      commands.present?
-    end
-
     def play(current_user)
       # Try to queue a current build
       if self.enqueue
@@ -131,8 +126,7 @@ module Ci
     end
 
     def retryable?
-      project.builds_enabled? && has_commands? &&
-        (success? || failed? || canceled?)
+      success? || failed? || canceled?
     end
 
     def retried?
@@ -171,19 +165,6 @@ module Ci
       latest_builds.where('stage_idx < ?', stage_idx)
     end
 
-    def trace_html(**args)
-      trace_with_state(**args)[:html] || ''
-    end
-
-    def trace_with_state(state: nil, last_lines: nil)
-      trace_ansi = trace(last_lines: last_lines)
-      if trace_ansi.present?
-        Ci::Ansi2html.convert(trace_ansi, state)
-      else
-        {}
-      end
-    end
-
     def timeout
       project.build_timeout
     end
@@ -244,136 +225,35 @@ module Ci
     end
 
     def update_coverage
-      coverage = extract_coverage(trace, coverage_regex)
+      coverage = trace.extract_coverage(coverage_regex)
       update_attributes(coverage: coverage) if coverage.present?
     end
 
-    def extract_coverage(text, regex)
-      return unless regex
-
-      matches = text.scan(Regexp.new(regex)).last
-      matches = matches.last if matches.is_a?(Array)
-      coverage = matches.gsub(/\d+(\.\d+)?/).first
-
-      if coverage.present?
-        coverage.to_f
-      end
-    rescue
-      # if bad regex or something goes wrong we dont want to interrupt transition
-      # so we just silentrly ignore error for now
-    end
-
-    def has_trace_file?
-      File.exist?(path_to_trace) || has_old_trace_file?
+    def trace
+      Gitlab::Ci::Trace.new(self)
     end
 
     def has_trace?
-      raw_trace.present?
-    end
-
-    def raw_trace(last_lines: nil)
-      if File.exist?(trace_file_path)
-        Gitlab::Ci::TraceReader.new(trace_file_path).
-          read(last_lines: last_lines)
-      else
-        # backward compatibility
-        read_attribute :trace
-      end
+      trace.exist?
     end
 
-    ##
-    # Deprecated
-    #
-    # This is a hotfix for CI build data integrity, see #4246
-    def has_old_trace_file?
-      project.ci_id && File.exist?(old_path_to_trace)
+    def trace=(data)
+      raise NotImplementedError
     end
 
-    def trace(last_lines: nil)
-      hide_secrets(raw_trace(last_lines: last_lines))
+    def old_trace
+      read_attribute(:trace)
     end
 
-    def trace_length
-      if raw_trace
-        raw_trace.bytesize
-      else
-        0
-      end
-    end
-
-    def trace=(trace)
-      recreate_trace_dir
-      trace = hide_secrets(trace)
-      File.write(path_to_trace, trace)
-    end
-
-    def recreate_trace_dir
-      unless Dir.exist?(dir_to_trace)
-        FileUtils.mkdir_p(dir_to_trace)
-      end
-    end
-    private :recreate_trace_dir
-
-    def append_trace(trace_part, offset)
-      recreate_trace_dir
-      touch if needs_touch?
-
-      trace_part = hide_secrets(trace_part)
-
-      File.truncate(path_to_trace, offset) if File.exist?(path_to_trace)
-      File.open(path_to_trace, 'ab') do |f|
-        f.write(trace_part)
-      end
+    def erase_old_trace!
+      write_attribute(:trace, nil)
+      save
     end
 
     def needs_touch?
       Time.now - updated_at > 15.minutes.to_i
     end
 
-    def trace_file_path
-      if has_old_trace_file?
-        old_path_to_trace
-      else
-        path_to_trace
-      end
-    end
-
-    def dir_to_trace
-      File.join(
-        Settings.gitlab_ci.builds_path,
-        created_at.utc.strftime("%Y_%m"),
-        project.id.to_s
-      )
-    end
-
-    def path_to_trace
-      "#{dir_to_trace}/#{id}.log"
-    end
-
-    ##
-    # Deprecated
-    #
-    # This is a hotfix for CI build data integrity, see #4246
-    # Should be removed in 8.4, after CI files migration has been done.
-    #
-    def old_dir_to_trace
-      File.join(
-        Settings.gitlab_ci.builds_path,
-        created_at.utc.strftime("%Y_%m"),
-        project.ci_id.to_s
-      )
-    end
-
-    ##
-    # Deprecated
-    #
-    # This is a hotfix for CI build data integrity, see #4246
-    # Should be removed in 8.4, after CI files migration has been done.
-    #
-    def old_path_to_trace
-      "#{old_dir_to_trace}/#{id}.log"
-    end
-
     ##
     # Deprecated
     #
@@ -540,6 +420,8 @@ module Ci
     end
 
     def dependencies
+      return [] if empty_dependencies?
+
       depended_jobs = depends_on_builds
 
       return depended_jobs unless options[:dependencies].present?
@@ -549,6 +431,19 @@ module Ci
       end
     end
 
+    def empty_dependencies?
+      options[:dependencies]&.empty?
+    end
+
+    def hide_secrets(trace)
+      return unless trace
+
+      trace = trace.dup
+      Ci::MaskSecret.mask!(trace, project.runners_token) if project
+      Ci::MaskSecret.mask!(trace, token)
+      trace
+    end
+
     private
 
     def update_artifacts_size
@@ -560,7 +455,7 @@ module Ci
     end
 
     def erase_trace!
-      self.trace = nil
+      trace.erase!
     end
 
     def update_erased!(user = nil)
@@ -622,15 +517,6 @@ module Ci
       pipeline.config_processor.build_attributes(name)
     end
 
-    def hide_secrets(trace)
-      return unless trace
-
-      trace = trace.dup
-      Ci::MaskSecret.mask!(trace, project.runners_token) if project
-      Ci::MaskSecret.mask!(trace, token)
-      trace
-    end
-
     def update_project_statistics
       return unless project
 
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 49dec770096..445247f1b41 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -4,14 +4,25 @@ module Ci
     include HasStatus
     include Importable
     include AfterCommitQueue
+    include Presentable
 
     belongs_to :project
     belongs_to :user
+    belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline'
+
+    has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
+    has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
 
     has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id
     has_many :builds, foreign_key: :commit_id
     has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id
 
+    has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build'
+    has_many :retryable_builds, -> { latest.failed_or_canceled }, foreign_key: :commit_id, class_name: 'Ci::Build'
+    has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus'
+    has_many :manual_actions, -> { latest.manual_actions }, foreign_key: :commit_id, class_name: 'Ci::Build'
+    has_many :artifacts, -> { latest.with_artifacts_not_expired }, foreign_key: :commit_id, class_name: 'Ci::Build'
+
     delegate :id, to: :project, prefix: true
 
     validates :sha, presence: { unless: :importing? }
@@ -20,7 +31,6 @@ module Ci
     validate :valid_commit_sha, unless: :importing?
 
     after_create :keep_around_commits, unless: :importing?
-    after_create :refresh_build_status_cache
 
     state_machine :status, initial: :created do
       event :enqueue do
@@ -65,6 +75,10 @@ module Ci
         pipeline.update_duration
       end
 
+      before_transition canceled: any - [:canceled] do |pipeline|
+        pipeline.auto_canceled_by = nil
+      end
+
       after_transition [:created, :pending] => :running do |pipeline|
         pipeline.run_after_commit { PipelineMetricsWorker.perform_async(id) }
       end
@@ -82,6 +96,8 @@ module Ci
 
         pipeline.run_after_commit do
           PipelineHooksWorker.perform_async(id)
+          Ci::ExpirePipelineCacheService.new(project, nil)
+            .execute(pipeline)
         end
       end
 
@@ -160,10 +176,6 @@ module Ci
       end
     end
 
-    def artifacts
-      builds.latest.with_artifacts_not_expired.includes(project: [:namespace])
-    end
-
     def valid_commit_sha
       if self.sha == Gitlab::Git::BLANK_SHA
         self.errors.add(:sha, " cant be 00000000 (branch removal)")
@@ -200,27 +212,37 @@ module Ci
       !tag?
     end
 
-    def manual_actions
-      builds.latest.manual_actions.includes(project: [:namespace])
-    end
-
     def stuck?
-      builds.pending.includes(:project).any?(&:stuck?)
+      pending_builds.any?(&:stuck?)
     end
 
     def retryable?
-      builds.latest.failed_or_canceled.any?(&:retryable?)
+      retryable_builds.any?
     end
 
     def cancelable?
-      statuses.cancelable.any?
+      cancelable_statuses.any?
+    end
+
+    def auto_canceled?
+      canceled? && auto_canceled_by_id?
     end
 
     def cancel_running
-      Gitlab::OptimisticLocking.retry_lock(
-        statuses.cancelable) do |cancelable|
-          cancelable.find_each(&:cancel)
+      Gitlab::OptimisticLocking.retry_lock(cancelable_statuses) do |cancelable|
+        cancelable.find_each do |job|
+          yield(job) if block_given?
+          job.cancel
         end
+      end
+    end
+
+    def auto_cancel_running(pipeline)
+      update(auto_canceled_by: pipeline)
+
+      cancel_running do |job|
+        job.auto_canceled_by = pipeline
+      end
     end
 
     def retry_failed(current_user)
@@ -328,7 +350,6 @@ module Ci
         when 'manual' then block
         end
       end
-      refresh_build_status_cache
     end
 
     def predefined_variables
@@ -370,10 +391,6 @@ module Ci
         .fabricate!
     end
 
-    def refresh_build_status_cache
-      Ci::PipelineStatus.new(project, sha: sha, status: status).store_in_cache_if_needed
-    end
-
     private
 
     def pipeline_data
diff --git a/app/models/ci/pipeline_status.rb b/app/models/ci/pipeline_status.rb
deleted file mode 100644
index 048047d0e34..00000000000
--- a/app/models/ci/pipeline_status.rb
+++ /dev/null
@@ -1,86 +0,0 @@
-# This class is not backed by a table in the main database.
-# It loads the latest Pipeline for the HEAD of a repository, and caches that
-# in Redis.
-module Ci
-  class PipelineStatus
-    attr_accessor :sha, :status, :project, :loaded
-
-    delegate :commit, to: :project
-
-    def self.load_for_project(project)
-      new(project).tap do |status|
-        status.load_status
-      end
-    end
-
-    def initialize(project, sha: nil, status: nil)
-      @project = project
-      @sha = sha
-      @status = status
-    end
-
-    def has_status?
-      loaded? && sha.present? && status.present?
-    end
-
-    def load_status
-      return if loaded?
-
-      if has_cache?
-        load_from_cache
-      else
-        load_from_commit
-        store_in_cache
-      end
-
-      self.loaded = true
-    end
-
-    def load_from_commit
-      return unless commit
-
-      self.sha = commit.sha
-      self.status = commit.status
-    end
-
-    # We only cache the status for the HEAD commit of a project
-    # This status is rendered in project lists
-    def store_in_cache_if_needed
-      return unless sha
-      return delete_from_cache unless commit
-      store_in_cache if commit.sha == self.sha
-    end
-
-    def load_from_cache
-      Gitlab::Redis.with do |redis|
-        self.sha, self.status = redis.hmget(cache_key, :sha, :status)
-      end
-    end
-
-    def store_in_cache
-      Gitlab::Redis.with do |redis|
-        redis.mapped_hmset(cache_key, { sha: sha, status: status })
-      end
-    end
-
-    def delete_from_cache
-      Gitlab::Redis.with do |redis|
-        redis.del(cache_key)
-      end
-    end
-
-    def has_cache?
-      Gitlab::Redis.with do |redis|
-        redis.exists(cache_key)
-      end
-    end
-
-    def loaded?
-      self.loaded
-    end
-
-    def cache_key
-      "projects/#{project.id}/build_status"
-    end
-  end
-end
diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb
index cba1d81a861..2f64f70685a 100644
--- a/app/models/ci/trigger.rb
+++ b/app/models/ci/trigger.rb
@@ -7,12 +7,15 @@ module Ci
     belongs_to :project
     belongs_to :owner, class_name: "User"
 
-    has_many :trigger_requests, dependent: :destroy
+    has_many :trigger_requests
+    has_one :trigger_schedule, dependent: :destroy
 
     validates :token, presence: true, uniqueness: true
 
     before_validation :set_default_values
 
+    accepts_nested_attributes_for :trigger_schedule
+
     def set_default_values
       self.token = SecureRandom.hex(15) if self.token.blank?
     end
@@ -36,5 +39,9 @@ module Ci
     def can_access_project?
       self.owner_id.blank? || Ability.allowed?(self.owner, :create_build, project)
     end
+
+    def trigger_schedule
+      super || build_trigger_schedule(project: project)
+    end
   end
 end
diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb
new file mode 100644
index 00000000000..012a18eb439
--- /dev/null
+++ b/app/models/ci/trigger_schedule.rb
@@ -0,0 +1,41 @@
+module Ci
+  class TriggerSchedule < ActiveRecord::Base
+    extend Ci::Model
+    include Importable
+
+    acts_as_paranoid
+
+    belongs_to :project
+    belongs_to :trigger
+
+    validates :trigger, presence: { unless: :importing? }
+    validates :cron, unless: :importing_or_inactive?, cron: true, presence: { unless: :importing_or_inactive? }
+    validates :cron_timezone, cron_timezone: true, presence: { unless: :importing_or_inactive? }
+    validates :ref, presence: { unless: :importing_or_inactive? }
+
+    before_save :set_next_run_at
+
+    scope :active, -> { where(active: true) }
+
+    def importing_or_inactive?
+      importing? || !active?
+    end
+
+    def set_next_run_at
+      self.next_run_at = Gitlab::Ci::CronParser.new(cron, cron_timezone).next_time_from(Time.now)
+    end
+
+    def schedule_next_run!
+      save! # with set_next_run_at
+    rescue ActiveRecord::RecordInvalid
+      update_attribute(:next_run_at, nil) # update without validation
+    end
+
+    def real_next_run(
+        worker_cron: Settings.cron_jobs['trigger_schedule_worker']['cron'],
+        worker_time_zone: Time.zone.name)
+      Gitlab::Ci::CronParser.new(worker_cron, worker_time_zone)
+                            .next_time_from(next_run_at)
+    end
+  end
+end