Add latest changes from gitlab-org/gitlab@13-4-stable-ee

17 files changed, 231 insertions, 158 deletions

diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb
index f143736ddc1..073ef465e13 100644
--- a/app/services/ci/archive_trace_service.rb
+++ b/app/services/ci/archive_trace_service.rb
@@ -13,6 +13,7 @@ module Ci
       end
 
       job.trace.archive!
+      job.remove_pending_state!
 
       # TODO: Remove this logging once we confirmed new live trace architecture is functional.
       # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667.
diff --git a/app/services/ci/cancel_user_pipelines_service.rb b/app/services/ci/cancel_user_pipelines_service.rb
index bcafb6b4a35..3a8b5e91088 100644
--- a/app/services/ci/cancel_user_pipelines_service.rb
+++ b/app/services/ci/cancel_user_pipelines_service.rb
@@ -7,6 +7,10 @@ module Ci
     # https://gitlab.com/gitlab-org/gitlab/issues/32332
     def execute(user)
       user.pipelines.cancelable.find_each(&:cancel_running)
+
+      ServiceResponse.success(message: 'Pipeline canceled')
+    rescue ActiveRecord::StaleObjectError
+      ServiceResponse.error(message: 'Error canceling pipeline')
     end
     # rubocop: enable CodeReuse/ActiveRecord
   end
diff --git a/app/services/ci/create_cross_project_pipeline_service.rb b/app/services/ci/create_downstream_pipeline_service.rb
index 23207d809d4..0394cfb6119 100644
--- a/app/services/ci/create_cross_project_pipeline_service.rb
+++ b/app/services/ci/create_downstream_pipeline_service.rb
@@ -1,12 +1,16 @@
 # frozen_string_literal: true
 
 module Ci
-  # TODO: rename this (and worker) to CreateDownstreamPipelineService
-  class CreateCrossProjectPipelineService < ::BaseService
+  # Takes in input a Ci::Bridge job and creates a downstream pipeline
+  # (either multi-project or child pipeline) according to the Ci::Bridge
+  # specifications.
+  class CreateDownstreamPipelineService < ::BaseService
     include Gitlab::Utils::StrongMemoize
 
     DuplicateDownstreamPipelineError = Class.new(StandardError)
 
+    MAX_DESCENDANTS_DEPTH = 2
+
     def execute(bridge)
       @bridge = bridge
 
@@ -73,9 +77,16 @@ module Ci
 
       # TODO: Remove this condition if favour of model validation
       # https://gitlab.com/gitlab-org/gitlab/issues/38338
-      if @bridge.triggers_child_pipeline? && @bridge.pipeline.parent_pipeline.present?
-        @bridge.drop!(:bridge_pipeline_is_child_pipeline)
-        return false
+      if ::Gitlab::Ci::Features.child_of_child_pipeline_enabled?(project)
+        if has_max_descendants_depth?
+          @bridge.drop!(:reached_max_descendant_pipelines_depth)
+          return false
+        end
+      else
+        if @bridge.triggers_child_pipeline? && @bridge.pipeline.parent_pipeline.present?
+          @bridge.drop!(:bridge_pipeline_is_child_pipeline)
+          return false
+        end
       end
 
       unless can_create_downstream_pipeline?(target_ref)
@@ -106,5 +117,12 @@ module Ci
         @bridge.downstream_project
       end
     end
+
+    def has_max_descendants_depth?
+      return false unless @bridge.triggers_child_pipeline?
+
+      ancestors_of_new_child = @bridge.pipeline.base_and_ancestors(same_project: true)
+      ancestors_of_new_child.count > MAX_DESCENDANTS_DEPTH
+    end
   end
 end
diff --git a/app/services/ci/create_job_artifacts_service.rb b/app/services/ci/create_job_artifacts_service.rb
index cd3807e0495..1fe65898d55 100644
--- a/app/services/ci/create_job_artifacts_service.rb
+++ b/app/services/ci/create_job_artifacts_service.rb
@@ -2,6 +2,8 @@
 
 module Ci
   class CreateJobArtifactsService < ::BaseService
+    include Gitlab::Utils::UsageData
+
     ArtifactsExistError = Class.new(StandardError)
 
     LSIF_ARTIFACT_TYPE = 'lsif'
@@ -25,7 +27,7 @@ module Ci
 
       if lsif?(artifact_type)
         headers[:ProcessLsif] = true
-        headers[:ProcessLsifReferences] = Feature.enabled?(:code_navigation_references, project, default_enabled: true)
+        track_usage_event('i_source_code_code_intelligence', project.id)
       end
 
       success(headers: headers)
diff --git a/app/services/ci/create_web_ide_terminal_service.rb b/app/services/ci/create_web_ide_terminal_service.rb
index 4f1bf0447d2..a78281aed16 100644
--- a/app/services/ci/create_web_ide_terminal_service.rb
+++ b/app/services/ci/create_web_ide_terminal_service.rb
@@ -70,7 +70,7 @@ module Ci
     end
 
     def load_terminal_config!
-      result = ::Ci::WebIdeConfigService.new(project, current_user, sha: sha).execute
+      result = ::Ide::TerminalConfigService.new(project, current_user, sha: sha).execute
       raise TerminalCreationError, result[:message] if result[:status] != :success
 
       @terminal = result[:terminal]
diff --git a/app/services/ci/destroy_expired_job_artifacts_service.rb b/app/services/ci/destroy_expired_job_artifacts_service.rb
index 1fa8926faa1..ca6e60f819a 100644
--- a/app/services/ci/destroy_expired_job_artifacts_service.rb
+++ b/app/services/ci/destroy_expired_job_artifacts_service.rb
@@ -20,18 +20,18 @@ module Ci
     def execute
       in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do
         loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do
-          destroy_batch
+          destroy_batch(Ci::JobArtifact) || destroy_batch(Ci::PipelineArtifact)
         end
       end
     end
 
     private
 
-    def destroy_batch
-      artifact_batch = if Gitlab::Ci::Features.destroy_only_unlocked_expired_artifacts_enabled?
-                         Ci::JobArtifact.expired(BATCH_SIZE).unlocked
+    def destroy_batch(klass)
+      artifact_batch = if klass == Ci::JobArtifact
+                         klass.expired(BATCH_SIZE).unlocked
                        else
-                         Ci::JobArtifact.expired(BATCH_SIZE)
+                         klass.expired(BATCH_SIZE)
                        end
 
       artifacts = artifact_batch.to_a
diff --git a/app/services/ci/destroy_pipeline_service.rb b/app/services/ci/destroy_pipeline_service.rb
index 9aea20c45f7..1d9533ed76f 100644
--- a/app/services/ci/destroy_pipeline_service.rb
+++ b/app/services/ci/destroy_pipeline_service.rb
@@ -8,6 +8,10 @@ module Ci
       Ci::ExpirePipelineCacheService.new.execute(pipeline, delete: true)
 
       pipeline.destroy!
+
+      ServiceResponse.success(message: 'Pipeline not found')
+    rescue ActiveRecord::RecordNotFound
+      ServiceResponse.error(message: 'Pipeline not found')
     end
   end
 end
diff --git a/app/services/ci/generate_coverage_reports_service.rb b/app/services/ci/generate_coverage_reports_service.rb
index ebd1eaf0bad..063fb966183 100644
--- a/app/services/ci/generate_coverage_reports_service.rb
+++ b/app/services/ci/generate_coverage_reports_service.rb
@@ -12,7 +12,7 @@ module Ci
       {
         status: :parsed,
         key: key(base_pipeline, head_pipeline),
-        data: head_pipeline.coverage_reports.pick(merge_request.new_paths)
+        data: head_pipeline.pipeline_artifacts.find_with_code_coverage.present.for_files(merge_request.new_paths)
       }
     rescue => e
       Gitlab::ErrorTracking.track_exception(e, project_id: project.id)
diff --git a/app/services/ci/parse_dotenv_artifact_service.rb b/app/services/ci/parse_dotenv_artifact_service.rb
index fcbdc94c097..71b306864b2 100644
--- a/app/services/ci/parse_dotenv_artifact_service.rb
+++ b/app/services/ci/parse_dotenv_artifact_service.rb
@@ -54,7 +54,7 @@ module Ci
     end
 
     def scan_line!(line)
-      result = line.scan(/^(.*)=(.*)$/).last
+      result = line.scan(/^(.*?)=(.*)$/).last
 
       raise ParserError, 'Invalid Format' if result.nil?
 
diff --git a/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb b/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb
index d0aa8b04775..aeabbb99468 100644
--- a/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb
+++ b/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb
@@ -77,15 +77,8 @@ module Ci
         private
 
         def status_for_array(statuses, dag:)
-          # TODO: This is hack to support
-          # the same exact behaviour for Atomic and Legacy processing
-          # that DAG is blocked from executing if dependent is not "complete"
-          if dag && statuses.any? { |status| Ci::HasStatus::COMPLETED_STATUSES.exclude?(status[:status]) }
-            return 'pending'
-          end
-
           result = Gitlab::Ci::Status::Composite
-            .new(statuses)
+            .new(statuses, dag: dag)
             .status
           result || 'success'
         end
diff --git a/app/services/ci/pipelines/create_artifact_service.rb b/app/services/ci/pipelines/create_artifact_service.rb
new file mode 100644
index 00000000000..b7d334e436d
--- /dev/null
+++ b/app/services/ci/pipelines/create_artifact_service.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+module Ci
+  module Pipelines
+    class CreateArtifactService
+      def execute(pipeline)
+        return unless ::Gitlab::Ci::Features.coverage_report_view?(pipeline.project)
+        return unless pipeline.can_generate_coverage_reports?
+        return if pipeline.has_coverage_reports?
+
+        file = build_carrierwave_file(pipeline)
+
+        pipeline.pipeline_artifacts.create!(
+          project_id: pipeline.project_id,
+          file_type: :code_coverage,
+          file_format: :raw,
+          size: file["tempfile"].size,
+          file: file,
+          expire_at: Ci::PipelineArtifact::EXPIRATION_DATE.from_now
+        )
+      end
+
+      private
+
+      def build_carrierwave_file(pipeline)
+        CarrierWaveStringFile.new_file(
+          file_content: pipeline.coverage_reports.to_json,
+          filename: Ci::PipelineArtifact::DEFAULT_FILE_NAMES.fetch(:code_coverage),
+          content_type: 'application/json'
+        )
+      end
+    end
+  end
+end
diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb
index d84ef5fbb93..18bae26613f 100644
--- a/app/services/ci/process_pipeline_service.rb
+++ b/app/services/ci/process_pipeline_service.rb
@@ -37,10 +37,12 @@ module Ci
         .pluck(Arel.sql('MAX(id)'), 'name')
 
       # mark builds that are retried
-      pipeline.statuses.latest
-        .where(name: latest_statuses.map(&:second))
-        .where.not(id: latest_statuses.map(&:first))
-        .update_all(retried: true) if latest_statuses.any?
+      if latest_statuses.any?
+        pipeline.statuses.latest
+          .where(name: latest_statuses.map(&:second))
+          .where.not(id: latest_statuses.map(&:first))
+          .update_all(retried: true)
+      end
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb
index 60b3d28b0c5..6b2e6c245f3 100644
--- a/app/services/ci/retry_build_service.rb
+++ b/app/services/ci/retry_build_service.rb
@@ -2,17 +2,20 @@
 
 module Ci
   class RetryBuildService < ::BaseService
-    CLONE_ACCESSORS = %i[pipeline project ref tag options name
-                         allow_failure stage stage_id stage_idx trigger_request
-                         yaml_variables when environment coverage_regex
-                         description tag_list protected needs_attributes
-                         resource_group scheduling_type].freeze
+    def self.clone_accessors
+      %i[pipeline project ref tag options name
+         allow_failure stage stage_id stage_idx trigger_request
+         yaml_variables when environment coverage_regex
+         description tag_list protected needs_attributes
+         resource_group scheduling_type].freeze
+    end
 
     def execute(build)
       build.ensure_scheduling_type!
 
       reprocess!(build).tap do |new_build|
-        build.pipeline.mark_as_processable_after_stage(build.stage_idx)
+        mark_subsequent_stages_as_processable(build)
+        build.pipeline.reset_ancestor_bridges!
 
         Gitlab::OptimisticLocking.retry_lock(new_build, &:enqueue)
 
@@ -28,7 +31,7 @@ module Ci
         raise Gitlab::Access::AccessDeniedError
       end
 
-      attributes = CLONE_ACCESSORS.map do |attribute|
+      attributes = self.class.clone_accessors.map do |attribute|
         [attribute, build.public_send(attribute)] # rubocop:disable GitlabSecurity/PublicSend
       end.to_h
 
@@ -60,5 +63,13 @@ module Ci
       end
       build
     end
+
+    def mark_subsequent_stages_as_processable(build)
+      build.pipeline.processables.skipped.after_stage(build.stage_idx).find_each do |processable|
+        Gitlab::OptimisticLocking.retry_lock(processable, &:process)
+      end
+    end
   end
 end
+
+Ci::RetryBuildService.prepend_if_ee('EE::Ci::RetryBuildService')
diff --git a/app/services/ci/retry_pipeline_service.rb b/app/services/ci/retry_pipeline_service.rb
index 2f52f0a39c1..45244d16393 100644
--- a/app/services/ci/retry_pipeline_service.rb
+++ b/app/services/ci/retry_pipeline_service.rb
@@ -26,6 +26,8 @@ module Ci
         retry_optimistic_lock(skipped) { |build| build.process }
       end
 
+      pipeline.reset_ancestor_bridges!
+
       MergeRequests::AddTodoWhenBuildFailsService
         .new(project, current_user)
         .close_all(pipeline)
diff --git a/app/services/ci/update_build_state_service.rb b/app/services/ci/update_build_state_service.rb
new file mode 100644
index 00000000000..61e4c77c1e5
--- /dev/null
+++ b/app/services/ci/update_build_state_service.rb
@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Ci
+  class UpdateBuildStateService
+    Result = Struct.new(:status, keyword_init: true)
+
+    ACCEPT_TIMEOUT = 5.minutes.freeze
+
+    attr_reader :build, :params, :metrics
+
+    def initialize(build, params, metrics = ::Gitlab::Ci::Trace::Metrics.new)
+      @build = build
+      @params = params
+      @metrics = metrics
+    end
+
+    def execute
+      overwrite_trace! if has_trace?
+
+      if accept_request?
+        accept_build_state!
+      else
+        check_migration_state
+        update_build_state!
+      end
+    end
+
+    private
+
+    def accept_build_state!
+      if Time.current - ensure_pending_state.created_at > ACCEPT_TIMEOUT
+        metrics.increment_trace_operation(operation: :discarded)
+
+        return update_build_state!
+      end
+
+      build.trace_chunks.live.find_each do |chunk|
+        chunk.schedule_to_persist!
+      end
+
+      metrics.increment_trace_operation(operation: :accepted)
+
+      Result.new(status: 202)
+    end
+
+    def overwrite_trace!
+      metrics.increment_trace_operation(operation: :overwrite)
+
+      build.trace.set(params[:trace]) if Gitlab::Ci::Features.trace_overwrite?
+    end
+
+    def check_migration_state
+      return unless accept_available?
+
+      if has_chunks? && !live_chunks_pending?
+        metrics.increment_trace_operation(operation: :finalized)
+      end
+    end
+
+    def update_build_state!
+      case build_state
+      when 'running'
+        build.touch if build.needs_touch?
+
+        Result.new(status: 200)
+      when 'success'
+        build.success!
+
+        Result.new(status: 200)
+      when 'failed'
+        build.drop!(params[:failure_reason] || :unknown_failure)
+
+        Result.new(status: 200)
+      else
+        Result.new(status: 400)
+      end
+    end
+
+    def accept_available?
+      !build_running? && has_checksum? && chunks_migration_enabled?
+    end
+
+    def accept_request?
+      accept_available? && live_chunks_pending?
+    end
+
+    def build_state
+      params.dig(:state).to_s
+    end
+
+    def has_trace?
+      params.dig(:trace).present?
+    end
+
+    def has_checksum?
+      params.dig(:checksum).present?
+    end
+
+    def has_chunks?
+      build.trace_chunks.any?
+    end
+
+    def live_chunks_pending?
+      build.trace_chunks.live.any?
+    end
+
+    def build_running?
+      build_state == 'running'
+    end
+
+    def ensure_pending_state
+      Ci::BuildPendingState.create_or_find_by!(
+        build_id: build.id,
+        state: params.fetch(:state),
+        trace_checksum: params.fetch(:checksum),
+        failure_reason: params.dig(:failure_reason)
+      )
+    rescue ActiveRecord::RecordNotFound
+      metrics.increment_trace_operation(operation: :conflict)
+
+      build.pending_state
+    end
+
+    def chunks_migration_enabled?
+      ::Gitlab::Ci::Features.accept_trace?(build.project)
+    end
+  end
+end
diff --git a/app/services/ci/update_ci_ref_status_service.rb b/app/services/ci/update_ci_ref_status_service.rb
deleted file mode 100644
index 22cc43232cc..00000000000
--- a/app/services/ci/update_ci_ref_status_service.rb
+++ /dev/null
@@ -1,66 +0,0 @@
-# frozen_string_literal: true
-
-# NOTE: This class is unused and to be removed in 13.1~
-module Ci
-  class UpdateCiRefStatusService
-    include Gitlab::OptimisticLocking
-
-    attr_reader :pipeline
-
-    def initialize(pipeline)
-      @pipeline = pipeline
-    end
-
-    def call
-      save.tap { |success| after_save if success }
-    end
-
-    private
-
-    def save
-      might_insert = ref.new_record?
-
-      begin
-        retry_optimistic_lock(ref) do
-          next false if ref.persisted? &&
-            (ref.last_updated_by_pipeline_id || 0) > pipeline.id
-
-          ref.update(status: next_status(ref.status, pipeline.status),
-                     last_updated_by_pipeline: pipeline)
-        end
-      rescue ActiveRecord::RecordNotUnique
-        if might_insert
-          @ref = pipeline.reset.ref_status
-          might_insert = false
-          retry
-        else
-          raise
-        end
-      end
-    end
-
-    def next_status(ref_status, pipeline_status)
-      if ref_status == 'failed' && pipeline_status == 'success'
-        'fixed'
-      else
-        pipeline_status
-      end
-    end
-
-    def after_save
-      enqueue_pipeline_notification
-    end
-
-    def enqueue_pipeline_notification
-      PipelineNotificationWorker.perform_async(pipeline.id, ref_status: ref.status)
-    end
-
-    def ref
-      @ref ||= pipeline.ref_status || build_ref
-    end
-
-    def build_ref
-      Ci::Ref.new(ref: pipeline.ref, project: pipeline.project, tag: pipeline.tag)
-    end
-  end
-end
diff --git a/app/services/ci/web_ide_config_service.rb b/app/services/ci/web_ide_config_service.rb
deleted file mode 100644
index ade9132f419..00000000000
--- a/app/services/ci/web_ide_config_service.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-module Ci
-  class WebIdeConfigService < ::BaseService
-    include ::Gitlab::Utils::StrongMemoize
-
-    ValidationError = Class.new(StandardError)
-
-    WEBIDE_CONFIG_FILE = '.gitlab/.gitlab-webide.yml'.freeze
-
-    attr_reader :config, :config_content
-
-    def execute
-      check_access!
-      load_config_content!
-      load_config!
-
-      success(terminal: config.terminal_value)
-    rescue ValidationError => e
-      error(e.message)
-    end
-
-    private
-
-    def check_access!
-      unless can?(current_user, :download_code, project)
-        raise ValidationError, 'Insufficient permissions to read configuration'
-      end
-    end
-
-    def load_config_content!
-      @config_content = webide_yaml_from_repo
-
-      unless config_content
-        raise ValidationError, "Failed to load Web IDE config file '#{WEBIDE_CONFIG_FILE}' for #{params[:sha]}"
-      end
-    end
-
-    def load_config!
-      @config = Gitlab::WebIde::Config.new(config_content)
-
-      unless @config.valid?
-        raise ValidationError, @config.errors.first
-      end
-    rescue Gitlab::WebIde::Config::ConfigError => e
-      raise ValidationError, e.message
-    end
-
-    def webide_yaml_from_repo
-      gitlab_webide_yml_for(params[:sha])
-    rescue GRPC::NotFound, GRPC::Internal
-      nil
-    end
-
-    def gitlab_webide_yml_for(sha)
-      project.repository.blob_data_at(sha, WEBIDE_CONFIG_FILE)
-    end
-  end
-end