Merge branch 'master' of gitlab_gitlab:gitlab-org/gitlab-cerunner-metrics-extractor

9 files changed, 108 insertions, 33 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 982ba0c04a7..40903aab5d0 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -38,8 +38,10 @@ module Ci
     has_one :deployment, as: :deployable, class_name: 'Deployment'
     has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
     has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id
+    has_many :needs, class_name: 'Ci::BuildNeed', foreign_key: :build_id, inverse_of: :build
 
     has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent
+    has_many :job_variables, class_name: 'Ci::JobVariable', foreign_key: :job_id
 
     Ci::JobArtifact.file_types.each do |key, value|
       has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
@@ -48,6 +50,8 @@ module Ci
     has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build
 
     accepts_nested_attributes_for :runner_session
+    accepts_nested_attributes_for :job_variables
+    accepts_nested_attributes_for :needs
 
     delegate :url, to: :runner_session, prefix: true, allow_nil: true
     delegate :terminal_specification, to: :runner_session, allow_nil: true
@@ -117,6 +121,8 @@ module Ci
     scope :scheduled_actions, ->() { where(when: :delayed, status: COMPLETED_STATUSES + %i[scheduled]) }
     scope :ref_protected, -> { where(protected: true) }
     scope :with_live_trace, -> { where('EXISTS (?)', Ci::BuildTraceChunk.where('ci_builds.id = ci_build_trace_chunks.build_id').select(1)) }
+    scope :with_stale_live_trace, -> { with_live_trace.finished_before(12.hours.ago) }
+    scope :finished_before, -> (date) { finished.where('finished_at < ?', date) }
 
     scope :matches_tag_ids, -> (tag_ids) do
       matcher = ::ActsAsTaggableOn::Tagging
@@ -266,7 +272,7 @@ module Ci
           begin
             Ci::Build.retry(build, build.user)
           rescue Gitlab::Access::AccessDeniedError => ex
-            Rails.logger.error "Unable to auto-retry job #{build.id}: #{ex}"
+            Rails.logger.error "Unable to auto-retry job #{build.id}: #{ex}" # rubocop:disable Gitlab/RailsLogger
           end
         end
       end
@@ -331,10 +337,10 @@ module Ci
     end
 
     # rubocop: disable CodeReuse/ServiceClass
-    def play(current_user)
+    def play(current_user, job_variables_attributes = nil)
       Ci::PlayBuildService
         .new(project, current_user)
-        .execute(self)
+        .execute(self, job_variables_attributes)
     end
     # rubocop: enable CodeReuse/ServiceClass
 
@@ -380,7 +386,7 @@ module Ci
       return unless has_environment?
 
       strong_memoize(:expanded_environment_name) do
-        ExpandVariables.expand(environment, simple_variables)
+        ExpandVariables.expand(environment, -> { simple_variables })
       end
     end
 
@@ -432,6 +438,7 @@ module Ci
         Gitlab::Ci::Variables::Collection.new
           .concat(persisted_variables)
           .concat(scoped_variables)
+          .concat(job_variables)
           .concat(persisted_environment_variables)
           .to_runner_variables
       end
@@ -531,6 +538,14 @@ module Ci
       trace.exist?
     end
 
+    def has_live_trace?
+      trace.live_trace_exist?
+    end
+
+    def has_archived_trace?
+      trace.archived_trace_exist?
+    end
+
     def artifacts_file
       job_artifacts_archive&.file
     end
@@ -582,7 +597,7 @@ module Ci
     end
 
     def valid_token?(token)
-      self.token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.token)
+      self.token && ActiveSupport::SecurityUtils.secure_compare(token, self.token)
     end
 
     def has_tags?
@@ -706,11 +721,17 @@ module Ci
 
       depended_jobs = depends_on_builds
 
-      return depended_jobs unless options[:dependencies].present?
+      # find all jobs that are needed
+      if Feature.enabled?(:ci_dag_support, project, default_enabled: true) && needs.exists?
+        depended_jobs = depended_jobs.where(name: needs.select(:name))
+      end
 
-      depended_jobs.select do |job|
-        options[:dependencies].include?(job.name)
+      # find all jobs that are dependent on
+      if options[:dependencies].present?
+        depended_jobs = depended_jobs.where(name: options[:dependencies])
       end
+
+      depended_jobs
     end
 
     def empty_dependencies?
diff --git a/app/models/ci/build_need.rb b/app/models/ci/build_need.rb
new file mode 100644
index 00000000000..6531dfd332f
--- /dev/null
+++ b/app/models/ci/build_need.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Ci
+  class BuildNeed < ApplicationRecord
+    extend Gitlab::Ci::Model
+
+    belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id, inverse_of: :needs
+
+    validates :build, presence: true
+    validates :name, presence: true, length: { maximum: 128 }
+
+    scope :scoped_build, -> { where('ci_builds.id=ci_build_needs.build_id') }
+  end
+end
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index f80e98e5bca..e132cb045e2 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -176,6 +176,10 @@ module Ci
       end
     end
 
+    def self.archived_trace_exists_for?(job_id)
+      where(job_id: job_id).trace.take&.file&.file&.exists?
+    end
+
     private
 
     def file_format_adapter_class
diff --git a/app/models/ci/job_variable.rb b/app/models/ci/job_variable.rb
new file mode 100644
index 00000000000..862a0bc1299
--- /dev/null
+++ b/app/models/ci/job_variable.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Ci
+  class JobVariable < ApplicationRecord
+    extend Gitlab::Ci::Model
+    include NewHasVariable
+
+    belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id
+
+    alias_attribute :secret_value, :value
+
+    validates :key, uniqueness: { scope: :job_id }
+  end
+end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 20ca4a9ab24..0a943a33bbb 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -196,7 +196,7 @@ module Ci
       sql = 'CASE ci_pipelines.source WHEN (?) THEN 0 ELSE 1 END, ci_pipelines.id DESC'
       query = ApplicationRecord.send(:sanitize_sql_array, [sql, sources[:merge_request_event]]) # rubocop:disable GitlabSecurity/PublicSend
 
-      order(query)
+      order(Arel.sql(query))
     end
 
     scope :for_user, -> (user) { where(user: user) }
@@ -229,15 +229,15 @@ module Ci
     #
     # ref - The name (or names) of the branch(es)/tag(s) to limit the list of
     #       pipelines to.
+    # sha - The commit SHA (or mutliple SHAs) to limit the list of pipelines to.
     # limit - This limits a backlog search, default to 100.
-    def self.newest_first(ref: nil, limit: 100)
+    def self.newest_first(ref: nil, sha: nil, limit: 100)
       relation = order(id: :desc)
       relation = relation.where(ref: ref) if ref
+      relation = relation.where(sha: sha) if sha
 
       if limit
         ids = relation.limit(limit).select(:id)
-        # MySQL does not support limit in subquery
-        ids = ids.pluck(:id) if Gitlab::Database.mysql?
         relation = relation.where(id: ids)
       end
 
@@ -248,10 +248,14 @@ module Ci
       newest_first(ref: ref).pluck(:status).first
     end
 
-    def self.latest_successful_for(ref)
+    def self.latest_successful_for_ref(ref)
       newest_first(ref: ref).success.take
     end
 
+    def self.latest_successful_for_sha(sha)
+      newest_first(sha: sha).success.take
+    end
+
     def self.latest_successful_for_refs(refs)
       relation = newest_first(ref: refs).success
 
@@ -324,6 +328,10 @@ module Ci
       config_sources.values_at(:repository_source, :auto_devops_source, :unknown_source)
     end
 
+    def self.bridgeable_statuses
+      ::Ci::Pipeline::AVAILABLE_STATUSES - %w[created preparing pending]
+    end
+
     def stages_count
       statuses.select(:stage).distinct.count
     end
@@ -500,8 +508,9 @@ module Ci
       return [] unless config_processor
 
       strong_memoize(:stage_seeds) do
-        seeds = config_processor.stages_attributes.map do |attributes|
-          Gitlab::Ci::Pipeline::Seed::Stage.new(self, attributes)
+        seeds = config_processor.stages_attributes.inject([]) do |previous_stages, attributes|
+          seed = Gitlab::Ci::Pipeline::Seed::Stage.new(self, attributes, previous_stages)
+          previous_stages + [seed]
         end
 
         seeds.select(&:included?)
@@ -607,8 +616,8 @@ module Ci
     end
 
     # rubocop: disable CodeReuse/ServiceClass
-    def process!
-      Ci::ProcessPipelineService.new(project, user).execute(self)
+    def process!(trigger_build_ids = nil)
+      Ci::ProcessPipelineService.new(project, user).execute(self, trigger_build_ids)
     end
     # rubocop: enable CodeReuse/ServiceClass
 
diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb
index ba8cea0cea9..42d4e86fe8d 100644
--- a/app/models/ci/pipeline_schedule.rb
+++ b/app/models/ci/pipeline_schedule.rb
@@ -4,11 +4,8 @@ module Ci
   class PipelineSchedule < ApplicationRecord
     extend Gitlab::Ci::Model
     include Importable
-    include IgnorableColumn
     include StripAttribute
 
-    ignore_column :deleted_at
-
     belongs_to :project
     belongs_to :owner, class_name: 'User'
     has_one :last_pipeline, -> { order(id: :desc) }, class_name: 'Ci::Pipeline'
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index 07d00503861..1c1c7a5ae7a 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -23,13 +23,17 @@ module Ci
       project_type: 3
     }
 
-    RUNNER_QUEUE_EXPIRY_TIME = 60.minutes
     ONLINE_CONTACT_TIMEOUT = 1.hour
-    UPDATE_DB_RUNNER_INFO_EVERY = 40.minutes
+    RUNNER_QUEUE_EXPIRY_TIME = 1.hour
+
+    # This needs to be less than `ONLINE_CONTACT_TIMEOUT`
+    UPDATE_CONTACT_COLUMN_EVERY = (40.minutes..55.minutes).freeze
+
     AVAILABLE_TYPES_LEGACY = %w[specific shared].freeze
     AVAILABLE_TYPES = runner_types.keys.freeze
     AVAILABLE_STATUSES = %w[active paused online offline].freeze
     AVAILABLE_SCOPES = (AVAILABLE_TYPES_LEGACY + AVAILABLE_TYPES + AVAILABLE_STATUSES).freeze
+
     FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_timeout_human_readable].freeze
 
     ignore_column :is_shared
@@ -46,7 +50,7 @@ module Ci
 
     scope :active, -> { where(active: true) }
     scope :paused, -> { where(active: false) }
-    scope :online, -> { where('contacted_at > ?', contact_time_deadline) }
+    scope :online, -> { where('contacted_at > ?', online_contact_time_deadline) }
     # The following query using negation is cheaper than using `contacted_at <= ?`
     # because there are less runners online than have been created. The
     # resulting query is quickly finding online ones and then uses the regular
@@ -56,6 +60,8 @@ module Ci
     scope :offline, -> { where.not(id: online) }
     scope :ordered, -> { order(id: :desc) }
 
+    scope :with_recent_runner_queue, -> { where('contacted_at > ?', recent_queue_deadline) }
+
     # BACKWARD COMPATIBILITY: There are needed to maintain compatibility with `AVAILABLE_SCOPES` used by `lib/api/runners.rb`
     scope :deprecated_shared, -> { instance_type }
     scope :deprecated_specific, -> { project_type.or(group_type) }
@@ -137,10 +143,18 @@ module Ci
       fuzzy_search(query, [:token, :description])
     end
 
-    def self.contact_time_deadline
+    def self.online_contact_time_deadline
       ONLINE_CONTACT_TIMEOUT.ago
     end
 
+    def self.recent_queue_deadline
+      # we add queue expiry + online
+      # - contacted_at can be updated at any time within this interval
+      #   we have always accurate `contacted_at` but it is stored in Redis
+      #   and not persisted in database
+      (ONLINE_CONTACT_TIMEOUT + RUNNER_QUEUE_EXPIRY_TIME).ago
+    end
+
     def self.order_by(order)
       if order == 'contacted_asc'
         order_contacted_at_asc
@@ -174,7 +188,7 @@ module Ci
     end
 
     def online?
-      contacted_at && contacted_at > self.class.contact_time_deadline
+      contacted_at && contacted_at > self.class.online_contact_time_deadline
     end
 
     def status
@@ -264,7 +278,7 @@ module Ci
     private
 
     def cleanup_runner_queue
-      Gitlab::Redis::Queues.with do |redis|
+      Gitlab::Redis::SharedState.with do |redis|
         redis.del(runner_queue_key)
       end
     end
@@ -275,9 +289,7 @@ module Ci
 
     def persist_cached_data?
       # Use a random threshold to prevent beating DB updates.
-      # It generates a distribution between [40m, 80m].
-
-      contacted_at_max_age = UPDATE_DB_RUNNER_INFO_EVERY + Random.rand(UPDATE_DB_RUNNER_INFO_EVERY)
+      contacted_at_max_age = Random.rand(UPDATE_CONTACT_COLUMN_EVERY)
 
       real_contacted_at = read_attribute(:contacted_at)
       real_contacted_at.nil? ||
diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb
index 8927bb9bc18..8792c5cf98b 100644
--- a/app/models/ci/trigger.rb
+++ b/app/models/ci/trigger.rb
@@ -3,17 +3,15 @@
 module Ci
   class Trigger < ApplicationRecord
     extend Gitlab::Ci::Model
-    include IgnorableColumn
     include Presentable
 
-    ignore_column :deleted_at
-
     belongs_to :project
     belongs_to :owner, class_name: "User"
 
     has_many :trigger_requests
 
     validates :token, presence: true, uniqueness: true
+    validates :owner, presence: true, unless: :supports_legacy_tokens?
 
     before_validation :set_default_values
 
@@ -37,8 +35,13 @@ module Ci
       self.owner_id.blank?
     end
 
+    def supports_legacy_tokens?
+      Feature.enabled?(:use_legacy_pipeline_triggers, project)
+    end
+
     def can_access_project?
-      self.owner_id.blank? || Ability.allowed?(self.owner, :create_build, project)
+      supports_legacy_tokens? && legacy? ||
+        Ability.allowed?(self.owner, :create_build, project)
     end
   end
 end
diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb
index a77bbef0fca..760872d3e6b 100644
--- a/app/models/ci/variable.rb
+++ b/app/models/ci/variable.rb
@@ -6,6 +6,7 @@ module Ci
     include HasVariable
     include Presentable
     include Maskable
+    prepend HasEnvironmentScope
 
     belongs_to :project