Add latest changes from gitlab-org/gitlab@13-9-stable-eev13.9.0-rc42

15 files changed, 167 insertions, 89 deletions

diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb
index ef3891908f7..ca400cebe4e 100644
--- a/app/models/ci/bridge.rb
+++ b/app/models/ci/bridge.rb
@@ -27,7 +27,7 @@ module Ci
     # rubocop:enable Cop/ActiveRecordSerialize
 
     state_machine :status do
-      after_transition [:created, :manual] => :pending do |bridge|
+      after_transition [:created, :manual, :waiting_for_resource] => :pending do |bridge|
         next unless bridge.downstream_project
 
         bridge.run_after_commit do
@@ -156,6 +156,10 @@ module Ci
       false
     end
 
+    def any_unmet_prerequisites?
+      false
+    end
+
     def expanded_environment_name
     end
 
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 5e3f42d7c2c..db151126caf 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -20,7 +20,6 @@ module Ci
     belongs_to :runner
     belongs_to :trigger_request
     belongs_to :erased_by, class_name: 'User'
-    belongs_to :resource_group, class_name: 'Ci::ResourceGroup', inverse_of: :builds
     belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id
 
     RUNNER_FEATURES = {
@@ -38,7 +37,6 @@ module Ci
     DEGRADATION_THRESHOLD_VARIABLE_NAME = 'DEGRADATION_THRESHOLD'
 
     has_one :deployment, as: :deployable, class_name: 'Deployment'
-    has_one :resource, class_name: 'Ci::Resource', inverse_of: :build
     has_one :pending_state, class_name: 'Ci::BuildPendingState', inverse_of: :build
     has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
     has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, inverse_of: :build
@@ -230,27 +228,20 @@ module Ci
       end
 
       def with_preloads
-        preload(:job_artifacts_archive, :job_artifacts, project: [:namespace])
+        preload(:job_artifacts_archive, :job_artifacts, :tags, project: [:namespace])
       end
     end
 
     state_machine :status do
       event :enqueue do
-        transition [:created, :skipped, :manual, :scheduled] => :waiting_for_resource, if: :requires_resource?
         transition [:created, :skipped, :manual, :scheduled] => :preparing, if: :any_unmet_prerequisites?
       end
 
       event :enqueue_scheduled do
-        transition scheduled: :waiting_for_resource, if: :requires_resource?
         transition scheduled: :preparing, if: :any_unmet_prerequisites?
         transition scheduled: :pending
       end
 
-      event :enqueue_waiting_for_resource do
-        transition waiting_for_resource: :preparing, if: :any_unmet_prerequisites?
-        transition waiting_for_resource: :pending
-      end
-
       event :enqueue_preparing do
         transition preparing: :pending
       end
@@ -279,23 +270,6 @@ module Ci
         build.scheduled_at = build.options_scheduled_at
       end
 
-      before_transition any => :waiting_for_resource do |build|
-        build.waiting_for_resource_at = Time.current
-      end
-
-      before_transition on: :enqueue_waiting_for_resource do |build|
-        next unless build.requires_resource?
-
-        build.resource_group.assign_resource_to(build) # If false is returned, it stops the transition
-      end
-
-      after_transition any => :waiting_for_resource do |build|
-        build.run_after_commit do
-          Ci::ResourceGroups::AssignResourceFromResourceGroupWorker
-            .perform_async(build.resource_group_id)
-        end
-      end
-
       before_transition on: :enqueue_preparing do |build|
         !build.any_unmet_prerequisites? # If false is returned, it stops the transition
       end
@@ -328,16 +302,6 @@ module Ci
         end
       end
 
-      after_transition any => ::Ci::Build.completed_statuses do |build|
-        next unless build.resource_group_id.present?
-        next unless build.resource_group.release_resource_from(build)
-
-        build.run_after_commit do
-          Ci::ResourceGroups::AssignResourceFromResourceGroupWorker
-            .perform_async(build.resource_group_id)
-        end
-      end
-
       after_transition any => [:success, :failed, :canceled] do |build|
         build.run_after_commit do
           build.run_status_commit_hooks!
@@ -403,7 +367,7 @@ module Ci
 
     def detailed_status(current_user)
       Gitlab::Ci::Status::Build::Factory
-        .new(self, current_user)
+        .new(self.present, current_user)
         .fabricate!
     end
 
@@ -467,6 +431,11 @@ module Ci
       pipeline.builds.retried.where(name: self.name).count
     end
 
+    override :all_met_to_become_pending?
+    def all_met_to_become_pending?
+      super && !any_unmet_prerequisites?
+    end
+
     def any_unmet_prerequisites?
       prerequisites.present?
     end
@@ -501,10 +470,6 @@ module Ci
       end
     end
 
-    def requires_resource?
-      self.resource_group_id.present?
-    end
-
     def has_environment?
       environment.present?
     end
@@ -821,7 +786,9 @@ module Ci
     end
 
     def artifacts_file_for_type(type)
-      job_artifacts.find_by(file_type: Ci::JobArtifact.file_types[type])&.file
+      file_types = Ci::JobArtifact.associated_file_types_for(type)
+      file_types_ids = file_types&.map { |file_type| Ci::JobArtifact.file_types[file_type] }
+      job_artifacts.find_by(file_type: file_types_ids)&.file
     end
 
     def coverage_regex
@@ -941,19 +908,12 @@ module Ci
     end
 
     def collect_coverage_reports!(coverage_report)
-      project_path, worktree_paths = if Feature.enabled?(:smart_cobertura_parser, project)
-                                       # If the flag is disabled, we intentionally pass nil
-                                       # for both project_path and worktree_paths to fallback
-                                       # to the non-smart behavior of the parser
-                                       [project.full_path, pipeline.all_worktree_paths]
-                                     end
-
       each_report(Ci::JobArtifact::COVERAGE_REPORT_FILE_TYPES) do |file_type, blob|
         Gitlab::Ci::Parsers.fabricate!(file_type).parse!(
           blob,
           coverage_report,
-          project_path: project_path,
-          worktree_paths: worktree_paths
+          project_path: project.full_path,
+          worktree_paths: pipeline.all_worktree_paths
         )
       end
 
@@ -1122,7 +1082,6 @@ module Ci
     end
 
     def conditionally_allow_failure!(exit_code)
-      return unless ::Gitlab::Ci::Features.allow_failure_with_exit_codes_enabled?
       return unless exit_code
 
       if allowed_to_fail_with_code?(exit_code)
diff --git a/app/models/ci/build_dependencies.rb b/app/models/ci/build_dependencies.rb
index a6abeb517c1..b50ecf99439 100644
--- a/app/models/ci/build_dependencies.rb
+++ b/app/models/ci/build_dependencies.rb
@@ -103,7 +103,7 @@ module Ci
     end
 
     def valid_local?
-      return true if Feature.enabled?(:ci_disable_validates_dependencies)
+      return true unless Gitlab::Ci::Features.validate_build_dependencies?(project)
 
       local.all?(&:valid_dependency?)
     end
diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb
index ceefb6a8b8a..d4f9f78a1ac 100644
--- a/app/models/ci/build_trace_chunk.rb
+++ b/app/models/ci/build_trace_chunk.rb
@@ -77,6 +77,22 @@ module Ci
       end
 
       ##
+      # Sometime we need to ensure that the first read goes to a primary
+      # database, what is especially important in EE. This method does not
+      # change the behavior in CE.
+      #
+      def with_read_consistency(build, &block)
+        return yield unless consistent_reads_enabled?(build)
+
+        ::Gitlab::Database::Consistency
+          .with_read_consistency(&block)
+      end
+
+      def consistent_reads_enabled?(build)
+        Feature.enabled?(:gitlab_ci_trace_read_consistency, build.project, type: :development, default_enabled: true)
+      end
+
+      ##
       # Sometimes we do not want to read raw data. This method makes it easier
       # to find attributes that are just metadata excluding raw data.
       #
@@ -154,8 +170,8 @@ module Ci
       in_lock(lock_key, **lock_params) do # exclusive Redis lock is acquired first
         raise FailedToPersistDataError, 'Modifed build trace chunk detected' if has_changes_to_save?
 
-        self.reset.then do |chunk|     # we ensure having latest lock_version
-          chunk.unsafe_persist_data!   # we migrate the data and update data store
+        self.class.with_read_consistency(build) do
+          self.reset.then { |chunk| chunk.unsafe_persist_data! }
         end
       end
     rescue FailedToObtainLockError
diff --git a/app/models/ci/build_trace_chunks/fog.rb b/app/models/ci/build_trace_chunks/fog.rb
index 27b579bf428..cbf0c0a1696 100644
--- a/app/models/ci/build_trace_chunks/fog.rb
+++ b/app/models/ci/build_trace_chunks/fog.rb
@@ -14,15 +14,7 @@ module Ci
       end
 
       def set_data(model, new_data)
-        if Feature.enabled?(:ci_live_trace_use_fog_attributes, default_enabled: true)
-          files.create(create_attributes(model, new_data))
-        else
-          # TODO: Support AWS S3 server side encryption
-          files.create({
-            key: key(model),
-            body: new_data
-          })
-        end
+        files.create(create_attributes(model, new_data))
       end
 
       def append_data(model, new_data, offset)
diff --git a/app/models/ci/build_trace_section.rb b/app/models/ci/build_trace_section.rb
index 8be42eb48d6..5091e3ff04a 100644
--- a/app/models/ci/build_trace_section.rb
+++ b/app/models/ci/build_trace_section.rb
@@ -2,6 +2,7 @@
 
 module Ci
   class BuildTraceSection < ApplicationRecord
+    extend SuppressCompositePrimaryKeyWarning
     extend Gitlab::Ci::Model
 
     belongs_to :build, class_name: 'Ci::Build'
diff --git a/app/models/ci/daily_build_group_report_result.rb b/app/models/ci/daily_build_group_report_result.rb
index e9f3366b939..23c96e63724 100644
--- a/app/models/ci/daily_build_group_report_result.rb
+++ b/app/models/ci/daily_build_group_report_result.rb
@@ -9,14 +9,19 @@ module Ci
 
     belongs_to :last_pipeline, class_name: 'Ci::Pipeline', foreign_key: :last_pipeline_id
     belongs_to :project
+    belongs_to :group
 
     validates :data, json_schema: { filename: "daily_build_group_report_result_data" }
 
     scope :with_included_projects, -> { includes(:project) }
+    scope :by_ref_path, -> (ref_path) { where(ref_path: ref_path) }
     scope :by_projects, -> (ids) { where(project_id: ids) }
+    scope :by_group, -> (group_id) { where(group_id: group_id) }
     scope :with_coverage, -> { where("(data->'coverage') IS NOT NULL") }
     scope :with_default_branch, -> { where(default_branch: true) }
     scope :by_date, -> (start_date) { where(date: report_window(start_date)..Date.current) }
+    scope :by_dates, -> (start_date, end_date) { where(date: start_date..end_date) }
+    scope :ordered_by_date_and_group_name, -> { order(date: :desc, group_name: :asc) }
 
     store_accessor :data, :coverage
 
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index f13be3b3c86..f927111758a 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -19,6 +19,8 @@ module Ci
     NON_ERASABLE_FILE_TYPES = %w[trace].freeze
     TERRAFORM_REPORT_FILE_TYPES = %w[terraform].freeze
     UNSUPPORTED_FILE_TYPES = %i[license_management].freeze
+    SAST_REPORT_TYPES = %w[sast].freeze
+    SECRET_DETECTION_REPORT_TYPES = %w[secret_detection].freeze
     DEFAULT_FILE_NAMES = {
       archive: nil,
       metadata: nil,
@@ -150,6 +152,14 @@ module Ci
       with_file_types(REPORT_TYPES.keys.map(&:to_s))
     end
 
+    scope :sast_reports, -> do
+      with_file_types(SAST_REPORT_TYPES)
+    end
+
+    scope :secret_detection_reports, -> do
+      with_file_types(SECRET_DETECTION_REPORT_TYPES)
+    end
+
     scope :test_reports, -> do
       with_file_types(TEST_REPORT_FILE_TYPES)
     end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 88c7002b1b6..3be107ea2e1 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -251,6 +251,7 @@ module Ci
       after_transition any => ::Ci::Pipeline.completed_statuses do |pipeline|
         pipeline.run_after_commit do
           ::Ci::PipelineArtifacts::CoverageReportWorker.perform_async(pipeline.id)
+          ::Ci::PipelineArtifacts::CreateQualityReportWorker.perform_async(pipeline.id)
         end
       end
 
@@ -263,8 +264,6 @@ module Ci
       end
 
       after_transition any => any do |pipeline|
-        next unless Feature.enabled?(:jira_sync_builds, pipeline.project)
-
         pipeline.run_after_commit do
           # Passing the seq-id ensures this is idempotent
           seq_id = ::Atlassian::JiraConnect::Client.generate_update_sequence_id
@@ -678,7 +677,7 @@ module Ci
 
     def number_of_warnings
       BatchLoader.for(id).batch(default_value: 0) do |pipeline_ids, loader|
-        ::Ci::Build.where(commit_id: pipeline_ids)
+        ::CommitStatus.where(commit_id: pipeline_ids)
           .latest
           .failed_but_allowed
           .group(:commit_id)
@@ -805,7 +804,7 @@ module Ci
           variables.concat(merge_request.predefined_variables)
         end
 
-        if Gitlab::Ci::Features.pipeline_open_merge_requests?(project) && open_merge_requests_refs.any?
+        if open_merge_requests_refs.any?
           variables.append(key: 'CI_OPEN_MERGE_REQUESTS', value: open_merge_requests_refs.join(','))
         end
 
@@ -962,7 +961,7 @@ module Ci
 
     def detailed_status(current_user)
       Gitlab::Ci::Status::Pipeline::Factory
-        .new(self, current_user)
+        .new(self.present, current_user)
         .fabricate!
     end
 
@@ -998,13 +997,23 @@ module Ci
     end
 
     def has_coverage_reports?
-      pipeline_artifacts&.has_code_coverage?
+      pipeline_artifacts&.report_exists?(:code_coverage)
     end
 
     def can_generate_coverage_reports?
       has_reports?(Ci::JobArtifact.coverage_reports)
     end
 
+    def has_codequality_mr_diff_report?
+      pipeline_artifacts&.report_exists?(:code_quality_mr_diff)
+    end
+
+    def can_generate_codequality_reports?
+      return false unless ::Gitlab::Ci::Features.display_quality_on_mr_diff?(project)
+
+      has_reports?(Ci::JobArtifact.codequality_reports)
+    end
+
     def test_report_summary
       strong_memoize(:test_report_summary) do
         Gitlab::Ci::Reports::TestReportSummary.new(latest_builds_report_results)
@@ -1206,6 +1215,21 @@ module Ci
     end
     # rubocop:enable Rails/FindEach
 
+    # EE-only
+    def merge_train_pipeline?
+      false
+    end
+
+    def security_reports(report_types: [])
+      reports_scope = report_types.empty? ? ::Ci::JobArtifact.security_reports : ::Ci::JobArtifact.security_reports(file_types: report_types)
+
+      ::Gitlab::Ci::Reports::Security::Reports.new(self).tap do |security_reports|
+        latest_report_builds(reports_scope).each do |build|
+          build.collect_security_reports!(security_reports)
+        end
+      end
+    end
+
     private
 
     def add_message(severity, content)
diff --git a/app/models/ci/pipeline_artifact.rb b/app/models/ci/pipeline_artifact.rb
index b6db8cad667..f538a4cd808 100644
--- a/app/models/ci/pipeline_artifact.rb
+++ b/app/models/ci/pipeline_artifact.rb
@@ -14,7 +14,13 @@ module Ci
     EXPIRATION_DATE = 1.week.freeze
 
     DEFAULT_FILE_NAMES = {
-      code_coverage: 'code_coverage.json'
+      code_coverage: 'code_coverage.json',
+      code_quality_mr_diff: 'code_quality_mr_diff.json'
+    }.freeze
+
+    REPORT_TYPES = {
+      code_coverage: :raw,
+      code_quality_mr_diff: :raw
     }.freeze
 
     belongs_to :project, class_name: "Project", inverse_of: :pipeline_artifacts
@@ -30,15 +36,20 @@ module Ci
     update_project_statistics project_statistics_name: :pipeline_artifacts_size
 
     enum file_type: {
-      code_coverage: 1
+      code_coverage: 1,
+      code_quality_mr_diff: 2
     }
 
-    def self.has_code_coverage?
-      where(file_type: :code_coverage).exists?
-    end
+    class << self
+      def report_exists?(file_type)
+        return false unless REPORT_TYPES.key?(file_type)
+
+        where(file_type: file_type).exists?
+      end
 
-    def self.find_with_code_coverage
-      find_by(file_type: :code_coverage)
+      def find_by_file_type(file_type)
+        find_by(file_type: file_type)
+      end
     end
 
     def present
diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb
index 8c9ad343f32..2fae077dd87 100644
--- a/app/models/ci/pipeline_schedule.rb
+++ b/app/models/ci/pipeline_schedule.rb
@@ -21,7 +21,7 @@ module Ci
     validates :cron_timezone, cron_timezone: true, presence: { unless: :importing? }
     validates :ref, presence: { unless: :importing? }
     validates :description, presence: true
-    validates :variables, variable_duplicates: true
+    validates :variables, nested_attributes_duplicates: true
 
     strip_attributes :cron
 
diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb
index 6aaf6ac530b..fae65ed0632 100644
--- a/app/models/ci/processable.rb
+++ b/app/models/ci/processable.rb
@@ -3,6 +3,11 @@
 module Ci
   class Processable < ::CommitStatus
     include Gitlab::Utils::StrongMemoize
+    extend ::Gitlab::Utils::Override
+
+    has_one :resource, class_name: 'Ci::Resource', foreign_key: 'build_id', inverse_of: :processable
+
+    belongs_to :resource_group, class_name: 'Ci::ResourceGroup', inverse_of: :processables
 
     accepts_nested_attributes_for :needs
 
@@ -20,6 +25,48 @@ module Ci
       where('NOT EXISTS (?)', needs)
     end
 
+    state_machine :status do
+      event :enqueue do
+        transition [:created, :skipped, :manual, :scheduled] => :waiting_for_resource, if: :with_resource_group?
+      end
+
+      event :enqueue_scheduled do
+        transition scheduled: :waiting_for_resource, if: :with_resource_group?
+      end
+
+      event :enqueue_waiting_for_resource do
+        transition waiting_for_resource: :preparing, if: :any_unmet_prerequisites?
+        transition waiting_for_resource: :pending
+      end
+
+      before_transition any => :waiting_for_resource do |processable|
+        processable.waiting_for_resource_at = Time.current
+      end
+
+      before_transition on: :enqueue_waiting_for_resource do |processable|
+        next unless processable.with_resource_group?
+
+        processable.resource_group.assign_resource_to(processable)
+      end
+
+      after_transition any => :waiting_for_resource do |processable|
+        processable.run_after_commit do
+          Ci::ResourceGroups::AssignResourceFromResourceGroupWorker
+            .perform_async(processable.resource_group_id)
+        end
+      end
+
+      after_transition any => ::Ci::Processable.completed_statuses do |processable|
+        next unless processable.with_resource_group?
+        next unless processable.resource_group.release_resource_from(processable)
+
+        processable.run_after_commit do
+          Ci::ResourceGroups::AssignResourceFromResourceGroupWorker
+            .perform_async(processable.resource_group_id)
+        end
+      end
+    end
+
     def self.select_with_aggregated_needs(project)
       aggregated_needs_names = Ci::BuildNeed
         .scoped_build
@@ -77,6 +124,15 @@ module Ci
       raise NotImplementedError
     end
 
+    override :all_met_to_become_pending?
+    def all_met_to_become_pending?
+      super && !with_resource_group?
+    end
+
+    def with_resource_group?
+      self.resource_group_id.present?
+    end
+
     # Overriding scheduling_type enum's method for nil `scheduling_type`s
     def scheduling_type_dag?
       scheduling_type.nil? ? find_legacy_scheduling_type == :dag : super
diff --git a/app/models/ci/resource.rb b/app/models/ci/resource.rb
index ee5b6546165..e0e1fab642d 100644
--- a/app/models/ci/resource.rb
+++ b/app/models/ci/resource.rb
@@ -5,9 +5,9 @@ module Ci
     extend Gitlab::Ci::Model
 
     belongs_to :resource_group, class_name: 'Ci::ResourceGroup', inverse_of: :resources
-    belongs_to :build, class_name: 'Ci::Build', inverse_of: :resource
+    belongs_to :processable, class_name: 'Ci::Processable', foreign_key: 'build_id', inverse_of: :resource
 
-    scope :free, -> { where(build: nil) }
-    scope :retained_by, -> (build) { where(build: build) }
+    scope :free, -> { where(processable: nil) }
+    scope :retained_by, -> (processable) { where(processable: processable) }
   end
 end
diff --git a/app/models/ci/resource_group.rb b/app/models/ci/resource_group.rb
index eb18f3da0bf..85fbe03e1c9 100644
--- a/app/models/ci/resource_group.rb
+++ b/app/models/ci/resource_group.rb
@@ -7,7 +7,7 @@ module Ci
     belongs_to :project, inverse_of: :resource_groups
 
     has_many :resources, class_name: 'Ci::Resource', inverse_of: :resource_group
-    has_many :builds, class_name: 'Ci::Build', inverse_of: :resource_group
+    has_many :processables, class_name: 'Ci::Processable', inverse_of: :resource_group
 
     validates :key,
       length: { maximum: 255 },
@@ -19,12 +19,12 @@ module Ci
     ##
     # NOTE: This is concurrency-safe method that the subquery in the `UPDATE`
     # works as explicit locking.
-    def assign_resource_to(build)
-      resources.free.limit(1).update_all(build_id: build.id) > 0
+    def assign_resource_to(processable)
+      resources.free.limit(1).update_all(build_id: processable.id) > 0
     end
 
-    def release_resource_from(build)
-      resources.retained_by(build).update_all(build_id: nil) > 0
+    def release_resource_from(processable)
+      resources.retained_by(processable).update_all(build_id: nil) > 0
     end
 
     private
diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb
index cc6bd1870b9..ae80692d598 100644
--- a/app/models/ci/stage.rb
+++ b/app/models/ci/stage.rb
@@ -118,7 +118,7 @@ module Ci
 
     def number_of_warnings
       BatchLoader.for(id).batch(default_value: 0) do |stage_ids, loader|
-        ::Ci::Build.where(stage_id: stage_ids)
+        ::CommitStatus.where(stage_id: stage_ids)
           .latest
           .failed_but_allowed
           .group(:stage_id)