Add latest changes from gitlab-org/gitlab@13-7-stable-eev13.7.0-rc42

6 files changed, 157 insertions, 48 deletions

diff --git a/app/services/ci/compare_codequality_reports_service.rb b/app/services/ci/compare_codequality_reports_service.rb
new file mode 100644
index 00000000000..20f5378f051
--- /dev/null
+++ b/app/services/ci/compare_codequality_reports_service.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Ci
+  class CompareCodequalityReportsService < CompareReportsBaseService
+    def comparer_class
+      Gitlab::Ci::Reports::CodequalityReportsComparer
+    end
+
+    def serializer_class
+      CodequalityReportsComparerSerializer
+    end
+
+    def get_report(pipeline)
+      pipeline&.codequality_reports
+    end
+  end
+end
diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb
index e3bab2de44e..dbe81521cfc 100644
--- a/app/services/ci/create_pipeline_service.rb
+++ b/app/services/ci/create_pipeline_service.rb
@@ -18,6 +18,7 @@ module Ci
                 Gitlab::Ci::Pipeline::Chain::EvaluateWorkflowRules,
                 Gitlab::Ci::Pipeline::Chain::Seed,
                 Gitlab::Ci::Pipeline::Chain::Limit::Size,
+                Gitlab::Ci::Pipeline::Chain::Limit::Deployments,
                 Gitlab::Ci::Pipeline::Chain::Validate::External,
                 Gitlab::Ci::Pipeline::Chain::Populate,
                 Gitlab::Ci::Pipeline::Chain::StopDryRun,
@@ -90,7 +91,9 @@ module Ci
     # rubocop: enable Metrics/ParameterLists
 
     def execute!(*args, &block)
-      execute(*args, &block).tap do |pipeline|
+      source, params = args[0], Hash(args[1])
+
+      execute(source, **params, &block).tap do |pipeline|
         unless pipeline.persisted?
           raise CreateError, pipeline.full_error_messages
         end
diff --git a/app/services/ci/list_config_variables_service.rb b/app/services/ci/list_config_variables_service.rb
index 4a5b3a92a2c..88dac514bb9 100644
--- a/app/services/ci/list_config_variables_service.rb
+++ b/app/services/ci/list_config_variables_service.rb
@@ -2,7 +2,26 @@
 
 module Ci
   class ListConfigVariablesService < ::BaseService
+    include ReactiveCaching
+
+    self.reactive_cache_key = ->(service) { [service.class.name, service.id] }
+    self.reactive_cache_work_type = :external_dependency
+    self.reactive_cache_worker_finder = ->(id, *_args) { from_cache(id) }
+
+    def self.from_cache(id)
+      project_id, user_id = id.split('-')
+
+      project = Project.find(project_id)
+      user = User.find(user_id)
+
+      new(project, user)
+    end
+
     def execute(sha)
+      with_reactive_cache(sha) { |result| result }
+    end
+
+    def calculate_reactive_cache(sha)
       config = project.ci_config_for(sha)
       return {} unless config
 
@@ -12,5 +31,10 @@ module Ci
 
       result.valid? ? result.variables_with_data : {}
     end
+
+    # Required for ReactiveCaching, it is also used in `reactive_cache_worker_finder`
+    def id
+      "#{project.id}-#{current_user.id}"
+    end
   end
 end
diff --git a/app/services/ci/test_cases_service.rb b/app/services/ci/test_cases_service.rb
deleted file mode 100644
index 3139b567571..00000000000
--- a/app/services/ci/test_cases_service.rb
+++ /dev/null
@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-module Ci
-  class TestCasesService
-    MAX_TRACKABLE_FAILURES = 200
-
-    def execute(build)
-      return unless Feature.enabled?(:test_failure_history, build.project)
-      return unless build.has_test_reports?
-      return unless build.project.default_branch_or_master == build.ref
-
-      test_suite = generate_test_suite_report(build)
-
-      track_failures(build, test_suite)
-    end
-
-    private
-
-    def generate_test_suite_report(build)
-      build.collect_test_reports!(Gitlab::Ci::Reports::TestReports.new)
-    end
-
-    def track_failures(build, test_suite)
-      return if test_suite.failed_count > MAX_TRACKABLE_FAILURES
-
-      test_suite.failed.keys.each_slice(100) do |keys|
-        Ci::TestCase.transaction do
-          test_cases = Ci::TestCase.find_or_create_by_batch(build.project, keys)
-          Ci::TestCaseFailure.insert_all(test_case_failures(test_cases, build))
-        end
-      end
-    end
-
-    def test_case_failures(test_cases, build)
-      test_cases.map do |test_case|
-        {
-          test_case_id: test_case.id,
-          build_id: build.id,
-          failed_at: build.finished_at
-        }
-      end
-    end
-  end
-end
diff --git a/app/services/ci/test_failure_history_service.rb b/app/services/ci/test_failure_history_service.rb
new file mode 100644
index 00000000000..99a2592ec06
--- /dev/null
+++ b/app/services/ci/test_failure_history_service.rb
@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Ci
+  class TestFailureHistoryService
+    class Async
+      attr_reader :service
+
+      def initialize(service)
+        @service = service
+      end
+
+      def perform_if_needed
+        TestFailureHistoryWorker.perform_async(service.pipeline.id) if service.should_track_failures?
+      end
+    end
+
+    MAX_TRACKABLE_FAILURES = 200
+
+    attr_reader :pipeline
+    delegate :project, to: :pipeline
+
+    def initialize(pipeline)
+      @pipeline = pipeline
+    end
+
+    def execute
+      return unless should_track_failures?
+
+      track_failures
+    end
+
+    def should_track_failures?
+      return false unless Feature.enabled?(:test_failure_history, project)
+      return false unless project.default_branch_or_master == pipeline.ref
+
+      # We fetch for up to MAX_TRACKABLE_FAILURES + 1 builds. So if ever we get
+      # 201 total number of builds with the assumption that each job has at least
+      # 1 failed test case, then we have at least 201 failed test cases which exceeds
+      # the MAX_TRACKABLE_FAILURES of 200. If this is the case, let's early exit so we
+      # don't have to parse each JUnit report of each of the 201 builds.
+      failed_builds.length <= MAX_TRACKABLE_FAILURES
+    end
+
+    def async
+      Async.new(self)
+    end
+
+    private
+
+    def failed_builds
+      @failed_builds ||= pipeline.builds_with_failed_tests(limit: MAX_TRACKABLE_FAILURES + 1)
+    end
+
+    def track_failures
+      failed_test_cases = gather_failed_test_cases(failed_builds)
+
+      return if failed_test_cases.size > MAX_TRACKABLE_FAILURES
+
+      failed_test_cases.keys.each_slice(100) do |key_hashes|
+        Ci::TestCase.transaction do
+          ci_test_cases = Ci::TestCase.find_or_create_by_batch(project, key_hashes)
+          failures = test_case_failures(ci_test_cases, failed_test_cases)
+
+          Ci::TestCaseFailure.insert_all(failures)
+        end
+      end
+    end
+
+    def gather_failed_test_cases(failed_builds)
+      failed_builds.each_with_object({}) do |build, failed_test_cases|
+        test_suite = generate_test_suite!(build)
+        test_suite.failed.keys.each do |key|
+          failed_test_cases[key] = build
+        end
+      end
+    end
+
+    def generate_test_suite!(build)
+      # Returns an instance of Gitlab::Ci::Reports::TestSuite
+      build.collect_test_reports!(Gitlab::Ci::Reports::TestReports.new)
+    end
+
+    def test_case_failures(ci_test_cases, failed_test_cases)
+      ci_test_cases.map do |test_case|
+        build = failed_test_cases[test_case.key_hash]
+
+        {
+          test_case_id: test_case.id,
+          build_id: build.id,
+          failed_at: build.finished_at
+        }
+      end
+    end
+  end
+end
diff --git a/app/services/ci/update_build_state_service.rb b/app/services/ci/update_build_state_service.rb
index fb67b0d2355..f01d41d9414 100644
--- a/app/services/ci/update_build_state_service.rb
+++ b/app/services/ci/update_build_state_service.rb
@@ -82,6 +82,10 @@ module Ci
         unless checksum.valid?
           metrics.increment_trace_operation(operation: :invalid)
 
+          if checksum.corrupted?
+            metrics.increment_trace_operation(operation: :corrupted)
+          end
+
           next unless log_invalid_chunks?
 
           ::Gitlab::ErrorTracking.log_exception(InvalidTraceError.new,
@@ -89,7 +93,8 @@ module Ci
             build_id: build.id,
             state_crc32: checksum.state_crc32,
             chunks_crc32: checksum.chunks_crc32,
-            chunks_count: checksum.chunks_count
+            chunks_count: checksum.chunks_count,
+            chunks_corrupted: checksum.corrupted?
           )
         end
       end
@@ -151,13 +156,21 @@ module Ci
     end
 
     def has_checksum?
-      params.dig(:checksum).present?
+      trace_checksum.present?
     end
 
     def build_running?
       build_state == 'running'
     end
 
+    def trace_checksum
+      params.dig(:output, :checksum) || params.dig(:checksum)
+    end
+
+    def trace_bytesize
+      params.dig(:output, :bytesize)
+    end
+
     def pending_state
       strong_memoize(:pending_state) { ensure_pending_state }
     end
@@ -166,7 +179,8 @@ module Ci
       build_state = Ci::BuildPendingState.safe_find_or_create_by(
         build_id: build.id,
         state: params.fetch(:state),
-        trace_checksum: params.fetch(:checksum),
+        trace_checksum: trace_checksum,
+        trace_bytesize: trace_bytesize,
         failure_reason: params.dig(:failure_reason)
       )