5 files changed, 273 insertions, 27 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index bf5f92f8462..5fe8ddf69d7 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -2,13 +2,24 @@ module Ci
   class Build < CommitStatus
     include TokenAuthenticatable
     include AfterCommitQueue
+    include Presentable
 
     belongs_to :runner
     belongs_to :trigger_request
     belongs_to :erased_by, class_name: 'User'
 
+    has_many :deployments, as: :deployable
+
+    # The "environment" field for builds is a String, and is the unexpanded name
+    def persisted_environment
+      @persisted_environment ||= Environment.find_by(
+        name: expanded_environment_name,
+        project_id: gl_project_id
+      )
+    end
+
     serialize :options
-    serialize :yaml_variables
+    serialize :yaml_variables, Gitlab::Serialize::Ci::Variables
 
     validates :coverage, numericality: true, allow_blank: true
     validates_presence_of :ref
@@ -33,6 +44,8 @@ module Ci
     before_destroy { project }
 
     after_create :execute_hooks
+    after_save :update_project_statistics, if: :artifacts_size_changed?
+    after_destroy :update_project_statistics
 
     class << self
       def first_pending
@@ -68,13 +81,23 @@ module Ci
           environment: build.environment,
           status_event: 'enqueue'
         )
-        MergeRequests::AddTodoWhenBuildFailsService.new(build.project, nil).close(new_build)
+
+        MergeRequests::AddTodoWhenBuildFailsService
+          .new(build.project, nil)
+          .close(new_build)
+
         build.pipeline.mark_as_processable_after_stage(build.stage_idx)
         new_build
       end
     end
 
     state_machine :status do
+      after_transition any => [:pending] do |build|
+        build.run_after_commit do
+          BuildQueueWorker.perform_async(id)
+        end
+      end
+
       after_transition pending: :running do |build|
         build.run_after_commit do
           BuildHooksWorker.perform_async(id)
@@ -94,6 +117,12 @@ module Ci
       end
     end
 
+    def detailed_status(current_user)
+      Gitlab::Ci::Status::Build::Factory
+        .new(self, current_user)
+        .fabricate!
+    end
+
     def manual?
       self.when == 'manual'
     end
@@ -117,14 +146,47 @@ module Ci
       end
     end
 
+    def cancelable?
+      active?
+    end
+
     def retryable?
-      project.builds_enabled? && commands.present? && complete?
+      project.builds_enabled? && commands.present? &&
+        (success? || failed? || canceled?)
     end
 
     def retried?
       !self.pipeline.statuses.latest.include?(self)
     end
 
+    def expanded_environment_name
+      ExpandVariables.expand(environment, simple_variables) if environment
+    end
+
+    def has_environment?
+      environment.present?
+    end
+
+    def starts_environment?
+      has_environment? && self.environment_action == 'start'
+    end
+
+    def stops_environment?
+      has_environment? && self.environment_action == 'stop'
+    end
+
+    def environment_action
+      self.options.fetch(:environment, {}).fetch(:action, 'start') if self.options
+    end
+
+    def outdated_deployment?
+      success? && !last_deployment.try(:last?)
+    end
+
+    def last_deployment
+      deployments.last
+    end
+
     def depends_on_builds
       # Get builds of the same type
       latest_builds = self.pipeline.builds.latest
@@ -150,12 +212,25 @@ module Ci
       project.build_timeout
     end
 
-    def variables
+    # A slugified version of the build ref, suitable for inclusion in URLs and
+    # domain names. Rules:
+    #
+    #   * Lowercased
+    #   * Anything not matching [a-z0-9-] is replaced with a -
+    #   * Maximum length is 63 bytes
+    def ref_slug
+      slugified = ref.to_s.downcase
+      slugified.gsub(/[^a-z0-9]/, '-')[0..62]
+    end
+
+    # Variables whose value does not depend on other variables
+    def simple_variables
       variables = predefined_variables
       variables += project.predefined_variables
       variables += pipeline.predefined_variables
       variables += runner.predefined_variables if runner
       variables += project.container_registry_variables
+      variables += project.deployment_variables if has_environment?
       variables += yaml_variables
       variables += user_variables
       variables += project.secret_variables
@@ -163,13 +238,20 @@ module Ci
       variables
     end
 
+    # All variables, including those dependent on other variables
+    def variables
+      variables = simple_variables
+      variables += persisted_environment.predefined_variables if persisted_environment.present?
+      variables
+    end
+
     def merge_request
       merge_requests = MergeRequest.includes(:merge_request_diff)
                                    .where(source_branch: ref, source_project_id: pipeline.gl_project_id)
                                    .reorder(iid: :asc)
 
       merge_requests.find do |merge_request|
-        merge_request.commits.any? { |ci| ci.id == pipeline.sha }
+        merge_request.commits_sha.include?(pipeline.sha)
       end
     end
 
@@ -271,6 +353,7 @@ module Ci
 
     def append_trace(trace_part, offset)
       recreate_trace_dir
+      touch if needs_touch?
 
       trace_part = hide_secrets(trace_part)
 
@@ -280,6 +363,10 @@ module Ci
       end
     end
 
+    def needs_touch?
+      Time.now - updated_at > 15.minutes.to_i
+    end
+
     def trace_file_path
       if has_old_trace_file?
         old_path_to_trace
@@ -427,6 +514,10 @@ module Ci
         end
     end
 
+    def has_expiring_artifacts?
+      artifacts_expire_at.present?
+    end
+
     def keep_artifacts!
       self.update(artifacts_expire_at: nil)
     end
@@ -448,6 +539,10 @@ module Ci
       ]
     end
 
+    def credentials
+      Gitlab::Ci::Build::Credentials::Factory.new(self).create!
+    end
+
     private
 
     def update_artifacts_size
@@ -475,6 +570,7 @@ module Ci
         { key: 'CI_BUILD_REF', value: sha, public: true },
         { key: 'CI_BUILD_BEFORE_SHA', value: before_sha, public: true },
         { key: 'CI_BUILD_REF_NAME', value: ref, public: true },
+        { key: 'CI_BUILD_REF_SLUG', value: ref_slug, public: true },
         { key: 'CI_BUILD_NAME', value: name, public: true },
         { key: 'CI_BUILD_STAGE', value: stage, public: true },
         { key: 'CI_SERVER_NAME', value: 'GitLab', public: true },
@@ -501,5 +597,9 @@ module Ci
       Ci::MaskSecret.mask!(trace, token)
       trace
     end
+
+    def update_project_statistics
+      ProjectCacheWorker.perform_async(project_id, [], [:build_artifacts_size])
+    end
   end
 end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 3fee6c18770..fab8497ec7d 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -21,8 +21,6 @@ module Ci
 
     after_create :keep_around_commits, unless: :importing?
 
-    delegate :stages, to: :statuses
-
     state_machine :status, initial: :created do
       event :enqueue do
         transition created: :pending
@@ -90,25 +88,66 @@ module Ci
     end
 
     # ref can't be HEAD or SHA, can only be branch/tag name
+    scope :latest, ->(ref = nil) do
+      max_id = unscope(:select)
+        .select("max(#{quoted_table_name}.id)")
+        .group(:ref, :sha)
+
+      relation = ref ? where(ref: ref) : self
+      relation.where(id: max_id)
+    end
+
+    def self.latest_status(ref = nil)
+      latest(ref).status
+    end
+
     def self.latest_successful_for(ref)
-      where(ref: ref).order(id: :desc).success.first
+      success.latest(ref).order(id: :desc).first
     end
 
     def self.truncate_sha(sha)
       sha[0...8]
     end
 
-    def self.stages
-      # We use pluck here due to problems with MySQL which doesn't allow LIMIT/OFFSET in queries
-      CommitStatus.where(pipeline: pluck(:id)).stages
-    end
-
     def self.total_duration
       where.not(duration: nil).sum(:duration)
     end
 
-    def stages_with_latest_statuses
-      statuses.latest.includes(project: :namespace).order(:stage_idx).group_by(&:stage)
+    def stage(name)
+      stage = Ci::Stage.new(self, name: name)
+      stage unless stage.statuses_count.zero?
+    end
+
+    def stages_count
+      statuses.select(:stage).distinct.count
+    end
+
+    def stages_name
+      statuses.order(:stage_idx).distinct.
+        pluck(:stage, :stage_idx).map(&:first)
+    end
+
+    def stages
+      # TODO, this needs refactoring, see gitlab-ce#26481.
+
+      stages_query = statuses
+        .group('stage').select(:stage).order('max(stage_idx)')
+
+      status_sql = statuses.latest.where('stage=sg.stage').status_sql
+
+      warnings_sql = statuses.latest.select('COUNT(*) > 0')
+        .where('stage=sg.stage').failed_but_allowed.to_sql
+
+      stages_with_statuses = CommitStatus.from(stages_query, :sg)
+        .pluck('sg.stage', status_sql, "(#{warnings_sql})")
+
+      stages_with_statuses.map do |stage|
+        Ci::Stage.new(self, Hash[%i[name status warnings].zip(stage)])
+      end
+    end
+
+    def artifacts
+      builds.latest.with_artifacts_not_expired.includes(project: [:namespace])
     end
 
     def project_id
@@ -157,27 +196,35 @@ module Ci
     end
 
     def manual_actions
-      builds.latest.manual_actions
+      builds.latest.manual_actions.includes(project: [:namespace])
+    end
+
+    def stuck?
+      builds.pending.any?(&:stuck?)
     end
 
     def retryable?
-      builds.latest.any? do |build|
-        (build.failed? || build.canceled?) && build.retryable?
-      end
+      builds.latest.failed_or_canceled.any?(&:retryable?)
     end
 
     def cancelable?
-      builds.running_or_pending.any?
+      statuses.cancelable.any?
     end
 
     def cancel_running
-      builds.running_or_pending.each(&:cancel)
+      Gitlab::OptimisticLocking.retry_lock(
+        statuses.cancelable) do |cancelable|
+          cancelable.each(&:cancel)
+        end
     end
 
     def retry_failed(user)
-      builds.latest.failed.select(&:retryable?).each do |build|
-        Ci::Build.retry(build, user)
-      end
+      Gitlab::OptimisticLocking.retry_lock(
+        builds.latest.failed_or_canceled) do |failed_or_canceled|
+          failed_or_canceled.select(&:retryable?).each do |build|
+            Ci::Build.retry(build, user)
+          end
+        end
     end
 
     def mark_as_processable_after_stage(stage_idx)
@@ -245,6 +292,10 @@ module Ci
       end
     end
 
+    def has_yaml_errors?
+      yaml_errors.present?
+    end
+
     def environments
       builds.where.not(environment: nil).success.pluck(:environment).uniq
     end
@@ -313,7 +364,13 @@ module Ci
     def merge_requests
       @merge_requests ||= project.merge_requests
         .where(source_branch: self.ref)
-        .select { |merge_request| merge_request.pipeline.try(:id) == self.id }
+        .select { |merge_request| merge_request.head_pipeline.try(:id) == self.id }
+    end
+
+    def detailed_status(current_user)
+      Gitlab::Ci::Status::Pipeline::Factory
+        .new(self, current_user)
+        .fabricate!
     end
 
     private
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index 123930273e0..ed1843ba005 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -2,6 +2,7 @@ module Ci
   class Runner < ActiveRecord::Base
     extend Ci::Model
 
+    RUNNER_QUEUE_EXPIRY_TIME = 60.minutes
     LAST_CONTACT_TIME = 1.hour.ago
     AVAILABLE_SCOPES = %w[specific shared active paused online]
     FORM_EDITABLE = %i[description tag_list active run_untagged locked]
@@ -21,6 +22,8 @@ module Ci
     scope :online, ->() { where('contacted_at > ?', LAST_CONTACT_TIME) }
     scope :ordered, ->() { order(id: :desc) }
 
+    after_save :tick_runner_queue, if: :form_editable_changed?
+
     scope :owned_or_shared, ->(project_id) do
       joins('LEFT JOIN ci_runner_projects ON ci_runner_projects.runner_id = ci_runners.id')
         .where("ci_runner_projects.gl_project_id = :project_id OR ci_runners.is_shared = true", project_id: project_id)
@@ -122,8 +125,38 @@ module Ci
       ]
     end
 
+    def tick_runner_queue
+      SecureRandom.hex.tap do |new_update|
+        Gitlab::Redis.with do |redis|
+          redis.set(runner_queue_key, new_update, ex: RUNNER_QUEUE_EXPIRY_TIME)
+        end
+      end
+    end
+
+    def ensure_runner_queue_value
+      Gitlab::Redis.with do |redis|
+        value = SecureRandom.hex
+        redis.set(runner_queue_key, value, ex: RUNNER_QUEUE_EXPIRY_TIME, nx: true)
+        redis.get(runner_queue_key)
+      end
+    end
+
+    def is_runner_queue_value_latest?(value)
+      ensure_runner_queue_value == value if value.present?
+    end
+
     private
 
+    def runner_queue_key
+      "runner:build_queue:#{self.token}"
+    end
+
+    def form_editable_changed?
+      FORM_EDITABLE.any? do |editable|
+        public_send("#{editable}_changed?")
+      end
+    end
+
     def tag_constraints
       unless has_tags? || run_untagged?
         errors.add(:tags_list,
diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb
new file mode 100644
index 00000000000..ca74c91b062
--- /dev/null
+++ b/app/models/ci/stage.rb
@@ -0,0 +1,56 @@
+module Ci
+  # Currently this is artificial object, constructed dynamically
+  # We should migrate this object to actual database record in the future
+  class Stage
+    include StaticModel
+
+    attr_reader :pipeline, :name
+
+    delegate :project, to: :pipeline
+
+    def initialize(pipeline, name:, status: nil, warnings: nil)
+      @pipeline = pipeline
+      @name = name
+      @status = status
+      @warnings = warnings
+    end
+
+    def to_param
+      name
+    end
+
+    def statuses_count
+      @statuses_count ||= statuses.count
+    end
+
+    def status
+      @status ||= statuses.latest.status
+    end
+
+    def detailed_status(current_user)
+      Gitlab::Ci::Status::Stage::Factory
+        .new(self, current_user)
+        .fabricate!
+    end
+
+    def statuses
+      @statuses ||= pipeline.statuses.where(stage: name)
+    end
+
+    def builds
+      @builds ||= pipeline.builds.where(stage: name)
+    end
+
+    def success?
+      status.to_s == 'success'
+    end
+
+    def has_warnings?
+      if @warnings.nil?
+        statuses.latest.failed_but_allowed.any?
+      else
+        @warnings
+      end
+    end
+  end
+end
diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb
index 94d9e2b3208..2c8698d8b5d 100644
--- a/app/models/ci/variable.rb
+++ b/app/models/ci/variable.rb
@@ -4,10 +4,10 @@ module Ci
 
     belongs_to :project, foreign_key: :gl_project_id
 
-    validates_uniqueness_of :key, scope: :gl_project_id
     validates :key,
       presence: true,
-      length: { within: 0..255 },
+      uniqueness: { scope: :gl_project_id },
+      length: { maximum: 255 },
       format: { with: /\A[a-zA-Z0-9_]+\z/,
                 message: "can contain only letters, digits and '_'." }