Add latest changes from gitlab-org/gitlab@14-4-stable-eev14.4.0-rc42

16 files changed, 461 insertions, 236 deletions

diff --git a/spec/services/ci/archive_trace_service_spec.rb b/spec/services/ci/archive_trace_service_spec.rb
index 071b5c3b2f9..b08ba6fd5e5 100644
--- a/spec/services/ci/archive_trace_service_spec.rb
+++ b/spec/services/ci/archive_trace_service_spec.rb
@@ -88,6 +88,32 @@ RSpec.describe Ci::ArchiveTraceService, '#execute' do
 
         subject
       end
+
+      context 'job has archive and chunks' do
+        let(:job) { create(:ci_build, :success, :trace_artifact) }
+
+        before do
+          create(:ci_build_trace_chunk, build: job, chunk_index: 0)
+        end
+
+        context 'archive is not completed' do
+          before do
+            job.job_artifacts_trace.file.remove!
+          end
+
+          it 'cleanups any stale archive data' do
+            expect(job.job_artifacts_trace).to be_present
+
+            subject
+
+            expect(job.reload.job_artifacts_trace).to be_nil
+          end
+        end
+
+        it 'removes trace chunks' do
+          expect { subject }.to change { job.trace_chunks.count }.to(0)
+        end
+      end
     end
 
     context 'when the archival process is backed off' do
diff --git a/spec/services/ci/create_pipeline_service/include_spec.rb b/spec/services/ci/create_pipeline_service/include_spec.rb
index 46271ee36c0..5e7dace8e15 100644
--- a/spec/services/ci/create_pipeline_service/include_spec.rb
+++ b/spec/services/ci/create_pipeline_service/include_spec.rb
@@ -58,17 +58,6 @@ RSpec.describe Ci::CreatePipelineService do
           expect(pipeline).to be_created_successfully
           expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec')
         end
-
-        context 'when the FF ci_include_rules is disabled' do
-          before do
-            stub_feature_flags(ci_include_rules: false)
-          end
-
-          it 'includes the job in the file' do
-            expect(pipeline).to be_created_successfully
-            expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec')
-          end
-        end
       end
 
       context 'when the rules does not match' do
@@ -78,17 +67,6 @@ RSpec.describe Ci::CreatePipelineService do
           expect(pipeline).to be_created_successfully
           expect(pipeline.processables.pluck(:name)).to contain_exactly('job')
         end
-
-        context 'when the FF ci_include_rules is disabled' do
-          before do
-            stub_feature_flags(ci_include_rules: false)
-          end
-
-          it 'includes the job in the file' do
-            expect(pipeline).to be_created_successfully
-            expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec')
-          end
-        end
       end
     end
   end
diff --git a/spec/services/ci/drop_pipeline_service_spec.rb b/spec/services/ci/drop_pipeline_service_spec.rb
index c6a118c6083..ddb53712d9c 100644
--- a/spec/services/ci/drop_pipeline_service_spec.rb
+++ b/spec/services/ci/drop_pipeline_service_spec.rb
@@ -50,13 +50,14 @@ RSpec.describe Ci::DropPipelineService do
       end.count
 
       writes_per_build = 2
+      load_balancer_queries = 3
       expected_reads_count = control_count - writes_per_build
 
       create_list(:ci_build, 5, :running, pipeline: cancelable_pipeline)
 
       expect do
         drop_pipeline!(cancelable_pipeline)
-      end.not_to exceed_query_limit(expected_reads_count + (5 * writes_per_build))
+      end.not_to exceed_query_limit(expected_reads_count + (5 * writes_per_build) + load_balancer_queries)
     end
   end
 end
diff --git a/spec/services/ci/pipelines/add_job_service_spec.rb b/spec/services/ci/pipelines/add_job_service_spec.rb
index 3a77d26dd9e..709a840c644 100644
--- a/spec/services/ci/pipelines/add_job_service_spec.rb
+++ b/spec/services/ci/pipelines/add_job_service_spec.rb
@@ -77,19 +77,6 @@ RSpec.describe Ci::Pipelines::AddJobService do
         expect(execute).to be_success
         expect(execute.payload[:job]).to eq(job)
       end
-
-      context 'when the FF ci_pipeline_add_job_with_lock is disabled' do
-        before do
-          stub_feature_flags(ci_pipeline_add_job_with_lock: false)
-        end
-
-        it 'does not use exclusive lock' do
-          expect(Gitlab::ExclusiveLease).not_to receive(:new).with(lock_key, timeout: lock_timeout)
-
-          expect(execute).to be_success
-          expect(execute.payload[:job]).to eq(job)
-        end
-      end
     end
   end
 end
diff --git a/spec/services/ci/pipelines/hook_service_spec.rb b/spec/services/ci/pipelines/hook_service_spec.rb
new file mode 100644
index 00000000000..0e1ef6afd0d
--- /dev/null
+++ b/spec/services/ci/pipelines/hook_service_spec.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::Pipelines::HookService do
+  describe '#execute_hooks' do
+    let_it_be(:namespace) { create(:namespace) }
+    let_it_be(:project) { create(:project, :repository, namespace: namespace) }
+    let_it_be(:pipeline) { create(:ci_empty_pipeline, :created, project: project) }
+
+    let(:hook_enabled) { true }
+    let!(:hook) { create(:project_hook, project: project, pipeline_events: hook_enabled) }
+    let(:hook_data) { double }
+
+    subject(:service) { described_class.new(pipeline) }
+
+    describe 'HOOK_NAME' do
+      specify { expect(described_class::HOOK_NAME).to eq(:pipeline_hooks) }
+    end
+
+    context 'with pipeline hooks enabled' do
+      before do
+        allow(Gitlab::DataBuilder::Pipeline).to receive(:build).with(pipeline).once.and_return(hook_data)
+      end
+
+      it 'calls pipeline.project.execute_hooks and pipeline.project.execute_integrations' do
+        create(:pipelines_email_integration, project: project)
+
+        expect(pipeline.project).to receive(:execute_hooks).with(hook_data, described_class::HOOK_NAME)
+        expect(pipeline.project).to receive(:execute_integrations).with(hook_data, described_class::HOOK_NAME)
+
+        service.execute
+      end
+    end
+
+    context 'with pipeline hooks and integrations disabled' do
+      let(:hook_enabled) { false }
+
+      it 'does not call pipeline.project.execute_hooks and pipeline.project.execute_integrations' do
+        expect(pipeline.project).not_to receive(:execute_hooks)
+        expect(pipeline.project).not_to receive(:execute_integrations)
+
+        service.execute
+      end
+    end
+  end
+end
diff --git a/spec/services/ci/play_bridge_service_spec.rb b/spec/services/ci/play_bridge_service_spec.rb
index 3f97bfdf5ae..56b1615a56d 100644
--- a/spec/services/ci/play_bridge_service_spec.rb
+++ b/spec/services/ci/play_bridge_service_spec.rb
@@ -23,18 +23,18 @@ RSpec.describe Ci::PlayBridgeService, '#execute' do
       expect(bridge.reload).to be_pending
     end
 
-    it 'enqueues Ci::CreateCrossProjectPipelineWorker' do
-      expect(::Ci::CreateCrossProjectPipelineWorker).to receive(:perform_async).with(bridge.id)
-
-      execute_service
-    end
-
     it "updates bridge's user" do
       execute_service
 
       expect(bridge.reload.user).to eq(user)
     end
 
+    it 'enqueues Ci::CreateDownstreamPipelineWorker' do
+      expect(::Ci::CreateDownstreamPipelineWorker).to receive(:perform_async).with(bridge.id)
+
+      execute_service
+    end
+
     context 'when a subsequent job is skipped' do
       let!(:job) { create(:ci_build, :skipped, pipeline: pipeline, stage_idx: bridge.stage_idx + 1) }
 
diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb
index b5bf0adadaf..404e1bf7c87 100644
--- a/spec/services/ci/process_pipeline_service_spec.rb
+++ b/spec/services/ci/process_pipeline_service_spec.rb
@@ -10,11 +10,9 @@ RSpec.describe Ci::ProcessPipelineService do
   end
 
   let(:pipeline_processing_events_counter) { double(increment: true) }
-  let(:legacy_update_jobs_counter) { double(increment: true) }
 
   let(:metrics) do
-    double(pipeline_processing_events_counter: pipeline_processing_events_counter,
-           legacy_update_jobs_counter: legacy_update_jobs_counter)
+    double(pipeline_processing_events_counter: pipeline_processing_events_counter)
   end
 
   subject { described_class.new(pipeline) }
@@ -33,68 +31,4 @@ RSpec.describe Ci::ProcessPipelineService do
       subject.execute
     end
   end
-
-  describe 'updating a list of retried builds' do
-    let!(:build_retried) { create_build('build') }
-    let!(:build) { create_build('build') }
-    let!(:test) { create_build('test') }
-
-    context 'when FF ci_remove_update_retried_from_process_pipeline is enabled' do
-      it 'does not update older builds as retried' do
-        subject.execute
-
-        expect(all_builds.latest).to contain_exactly(build, build_retried, test)
-        expect(all_builds.retried).to be_empty
-      end
-    end
-
-    context 'when FF ci_remove_update_retried_from_process_pipeline is disabled' do
-      before do
-        stub_feature_flags(ci_remove_update_retried_from_process_pipeline: false)
-      end
-
-      it 'returns unique statuses' do
-        subject.execute
-
-        expect(all_builds.latest).to contain_exactly(build, test)
-        expect(all_builds.retried).to contain_exactly(build_retried)
-      end
-
-      it 'increments the counter' do
-        expect(legacy_update_jobs_counter).to receive(:increment)
-
-        subject.execute
-      end
-
-      it 'logs the project and pipeline id' do
-        expect(Gitlab::AppJsonLogger).to receive(:info).with(event: 'update_retried_is_used',
-                                                             project_id: project.id,
-                                                             pipeline_id: pipeline.id)
-
-        subject.execute
-      end
-
-      context 'when the previous build has already retried column true' do
-        before do
-          build_retried.update_columns(retried: true)
-        end
-
-        it 'does not increment the counter' do
-          expect(legacy_update_jobs_counter).not_to receive(:increment)
-
-          subject.execute
-        end
-      end
-    end
-
-    private
-
-    def create_build(name, **opts)
-      create(:ci_build, :created, pipeline: pipeline, name: name, **opts)
-    end
-
-    def all_builds
-      pipeline.builds.order(:stage_idx, :id)
-    end
-  end
 end
diff --git a/spec/services/ci/register_job_service_spec.rb b/spec/services/ci/register_job_service_spec.rb
index 73ff15ec393..650353eb751 100644
--- a/spec/services/ci/register_job_service_spec.rb
+++ b/spec/services/ci/register_job_service_spec.rb
@@ -14,7 +14,7 @@ module Ci
     let!(:pending_job) { create(:ci_build, :pending, :queued, pipeline: pipeline) }
 
     describe '#execute' do
-      context 'checks database loadbalancing stickiness', :db_load_balancing do
+      context 'checks database loadbalancing stickiness' do
         subject { described_class.new(shared_runner).execute }
 
         before do
@@ -22,14 +22,14 @@ module Ci
         end
 
         it 'result is valid if replica did caught-up' do
-          expect(Gitlab::Database::LoadBalancing::Sticking).to receive(:all_caught_up?)
+          expect(ApplicationRecord.sticking).to receive(:all_caught_up?)
             .with(:runner, shared_runner.id) { true }
 
           expect(subject).to be_valid
         end
 
         it 'result is invalid if replica did not caught-up' do
-          expect(Gitlab::Database::LoadBalancing::Sticking).to receive(:all_caught_up?)
+          expect(ApplicationRecord.sticking).to receive(:all_caught_up?)
             .with(:runner, shared_runner.id) { false }
 
           expect(subject).not_to be_valid
@@ -87,19 +87,25 @@ module Ci
           end
 
           context 'for specific runner' do
-            context 'with FF disabled' do
+            context 'with tables decoupling disabled' do
               before do
                 stub_feature_flags(
                   ci_pending_builds_project_runners_decoupling: false,
                   ci_queueing_builds_enabled_checks: false)
               end
 
+              around do |example|
+                allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332952') do
+                  example.run
+                end
+              end
+
               it 'does not pick a build' do
                 expect(execute(specific_runner)).to be_nil
               end
             end
 
-            context 'with FF enabled' do
+            context 'with tables decoupling enabled' do
               before do
                 stub_feature_flags(
                   ci_pending_builds_project_runners_decoupling: true,
@@ -266,17 +272,23 @@ module Ci
           context 'and uses project runner' do
             let(:build) { execute(specific_runner) }
 
-            context 'with FF disabled' do
+            context 'with tables decoupling disabled' do
               before do
                 stub_feature_flags(
                   ci_pending_builds_project_runners_decoupling: false,
                   ci_queueing_builds_enabled_checks: false)
               end
 
+              around do |example|
+                allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332952') do
+                  example.run
+                end
+              end
+
               it { expect(build).to be_nil }
             end
 
-            context 'with FF enabled' do
+            context 'with tables decoupling enabled' do
               before do
                 stub_feature_flags(
                   ci_pending_builds_project_runners_decoupling: true,
@@ -791,6 +803,12 @@ module Ci
             stub_feature_flags(ci_queueing_denormalize_shared_runners_information: false)
           end
 
+          around do |example|
+            allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332952') do
+              example.run
+            end
+          end
+
           include_examples 'handles runner assignment'
         end
 
@@ -807,6 +825,12 @@ module Ci
             stub_feature_flags(ci_queueing_denormalize_tags_information: false)
           end
 
+          around do |example|
+            allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332952') do
+              example.run
+            end
+          end
+
           include_examples 'handles runner assignment'
         end
 
@@ -815,6 +839,12 @@ module Ci
             stub_feature_flags(ci_queueing_denormalize_namespace_traversal_ids: false)
           end
 
+          around do |example|
+            allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332952') do
+              example.run
+            end
+          end
+
           include_examples 'handles runner assignment'
         end
       end
@@ -824,6 +854,12 @@ module Ci
           stub_feature_flags(ci_pending_builds_queue_source: false)
         end
 
+        around do |example|
+          allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332952') do
+            example.run
+          end
+        end
+
         include_examples 'handles runner assignment'
       end
     end
diff --git a/spec/services/ci/resource_groups/assign_resource_from_resource_group_service_spec.rb b/spec/services/ci/resource_groups/assign_resource_from_resource_group_service_spec.rb
index 53aa842bc28..194203a422c 100644
--- a/spec/services/ci/resource_groups/assign_resource_from_resource_group_service_spec.rb
+++ b/spec/services/ci/resource_groups/assign_resource_from_resource_group_service_spec.rb
@@ -48,6 +48,92 @@ RSpec.describe Ci::ResourceGroups::AssignResourceFromResourceGroupService do
           expect(build).to be_pending
         end
       end
+
+      context 'when process mode is oldest_first' do
+        let(:resource_group) { create(:ci_resource_group, process_mode: :oldest_first, project: project) }
+
+        it 'requests resource' do
+          subject
+
+          expect(build.reload).to be_pending
+          expect(build.resource).to be_present
+        end
+
+        context 'when the other job exists in the newer pipeline' do
+          let!(:build_2) { create(:ci_build, :waiting_for_resource, project: project, user: user, resource_group: resource_group) }
+
+          it 'requests resource for the job in the oldest pipeline' do
+            subject
+
+            expect(build.reload).to be_pending
+            expect(build.resource).to be_present
+            expect(build_2.reload).to be_waiting_for_resource
+            expect(build_2.resource).to be_nil
+          end
+        end
+
+        context 'when build is not `waiting_for_resource` state' do
+          let!(:build) { create(:ci_build, :created, project: project, user: user, resource_group: resource_group) }
+
+          it 'attempts to request a resource' do
+            expect_next_found_instance_of(Ci::Build) do |job|
+              expect(job).to receive(:enqueue_waiting_for_resource).and_call_original
+            end
+
+            subject
+          end
+
+          it 'does not change the job status' do
+            subject
+
+            expect(build.reload).to be_created
+            expect(build.resource).to be_nil
+          end
+        end
+      end
+
+      context 'when process mode is newest_first' do
+        let(:resource_group) { create(:ci_resource_group, process_mode: :newest_first, project: project) }
+
+        it 'requests resource' do
+          subject
+
+          expect(build.reload).to be_pending
+          expect(build.resource).to be_present
+        end
+
+        context 'when the other job exists in the newer pipeline' do
+          let!(:build_2) { create(:ci_build, :waiting_for_resource, project: project, user: user, resource_group: resource_group) }
+
+          it 'requests resource for the job in the newest pipeline' do
+            subject
+
+            expect(build.reload).to be_waiting_for_resource
+            expect(build.resource).to be_nil
+            expect(build_2.reload).to be_pending
+            expect(build_2.resource).to be_present
+          end
+        end
+
+        context 'when build is not `waiting_for_resource` state' do
+          let!(:build) { create(:ci_build, :created, project: project, user: user, resource_group: resource_group) }
+
+          it 'attempts to request a resource' do
+            expect_next_found_instance_of(Ci::Build) do |job|
+              expect(job).to receive(:enqueue_waiting_for_resource).and_call_original
+            end
+
+            subject
+          end
+
+          it 'does not change the job status' do
+            subject
+
+            expect(build.reload).to be_created
+            expect(build.resource).to be_nil
+          end
+        end
+      end
     end
 
     context 'when there are no available resources' do
diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb
index ce2e6ba5e15..15c88c9f657 100644
--- a/spec/services/ci/retry_build_service_spec.rb
+++ b/spec/services/ci/retry_build_service_spec.rb
@@ -48,7 +48,7 @@ RSpec.describe Ci::RetryBuildService do
        job_artifacts_network_referee job_artifacts_dotenv
        job_artifacts_cobertura needs job_artifacts_accessibility
        job_artifacts_requirements job_artifacts_coverage_fuzzing
-       job_artifacts_api_fuzzing].freeze
+       job_artifacts_api_fuzzing terraform_state_versions].freeze
 
   ignore_accessors =
     %i[type lock_version target_url base_tags trace_sections
@@ -88,6 +88,7 @@ RSpec.describe Ci::RetryBuildService do
 
       create(:ci_job_variable, job: build)
       create(:ci_build_need, build: build)
+      create(:terraform_state_version, build: build)
     end
 
     describe 'clone accessors' do
@@ -276,13 +277,17 @@ RSpec.describe Ci::RetryBuildService do
     end
   end
 
-  describe '#reprocess' do
+  describe '#clone!' do
     let(:new_build) do
       travel_to(1.second.from_now) do
-        service.reprocess!(build)
+        service.clone!(build)
       end
     end
 
+    it 'raises an error when an unexpected class is passed' do
+      expect { service.clone!(create(:ci_build).present) }.to raise_error(TypeError)
+    end
+
     context 'when user has ability to execute build' do
       before do
         stub_not_protect_default_branch
@@ -338,7 +343,7 @@ RSpec.describe Ci::RetryBuildService do
       let(:user) { reporter }
 
       it 'raises an error' do
-        expect { service.reprocess!(build) }
+        expect { service.clone!(build) }
           .to raise_error Gitlab::Access::AccessDeniedError
       end
     end
diff --git a/spec/services/ci/retry_pipeline_service_spec.rb b/spec/services/ci/retry_pipeline_service_spec.rb
index 3e2e9f07723..12106b70969 100644
--- a/spec/services/ci/retry_pipeline_service_spec.rb
+++ b/spec/services/ci/retry_pipeline_service_spec.rb
@@ -316,6 +316,26 @@ RSpec.describe Ci::RetryPipelineService, '#execute' do
         expect(bridge.reload).to be_pending
       end
     end
+
+    context 'when there are skipped jobs in later stages' do
+      before do
+        create_build('build 1', :success, 0)
+        create_build('test 2', :failed, 1)
+        create_build('report 3', :skipped, 2)
+        create_bridge('deploy 4', :skipped, 2)
+      end
+
+      it 'retries failed jobs and processes skipped jobs' do
+        service.execute(pipeline)
+
+        expect(build('build 1')).to be_success
+        expect(build('test 2')).to be_pending
+        expect(build('report 3')).to be_created
+        expect(build('deploy 4')).to be_created
+
+        expect(pipeline.reload).to be_running
+      end
+    end
   end
 
   context 'when user is not allowed to retry pipeline' do
@@ -390,16 +410,25 @@ RSpec.describe Ci::RetryPipelineService, '#execute' do
     pipeline.reload.statuses
   end
 
+  # The method name can be confusing because this can actually return both Ci::Build and Ci::Bridge
   def build(name)
     statuses.latest.find_by(name: name)
   end
 
   def create_build(name, status, stage_num, **opts)
-    create(:ci_build, name: name,
-                      status: status,
-                      stage: "stage_#{stage_num}",
-                      stage_idx: stage_num,
-                      pipeline: pipeline, **opts) do |build|
+    create_processable(:ci_build, name, status, stage_num, **opts)
+  end
+
+  def create_bridge(name, status, stage_num, **opts)
+    create_processable(:ci_bridge, name, status, stage_num, **opts)
+  end
+
+  def create_processable(type, name, status, stage_num, **opts)
+    create(type, name: name,
+                 status: status,
+                 stage: "stage_#{stage_num}",
+                 stage_idx: stage_num,
+                 pipeline: pipeline, **opts) do |_job|
       ::Ci::ProcessPipelineService.new(pipeline).execute
     end
   end
diff --git a/spec/services/ci/stuck_builds/drop_service_spec.rb b/spec/services/ci/stuck_builds/drop_pending_service_spec.rb
index 8dfd1bc1b3d..aa0526edf57 100644
--- a/spec/services/ci/stuck_builds/drop_service_spec.rb
+++ b/spec/services/ci/stuck_builds/drop_pending_service_spec.rb
@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-RSpec.describe Ci::StuckBuilds::DropService do
+RSpec.describe Ci::StuckBuilds::DropPendingService do
   let!(:runner) { create :ci_runner }
   let!(:job) { create :ci_build, runner: runner }
   let(:created_at) { }
@@ -17,48 +17,6 @@ RSpec.describe Ci::StuckBuilds::DropService do
     job.update!(job_attributes)
   end
 
-  shared_examples 'job is dropped' do
-    it 'changes status' do
-      expect(service).to receive(:drop).exactly(3).times.and_call_original
-      expect(service).to receive(:drop_stuck).exactly(:once).and_call_original
-
-      service.execute
-      job.reload
-
-      expect(job).to be_failed
-      expect(job).to be_stuck_or_timeout_failure
-    end
-
-    context 'when job have data integrity problem' do
-      it "does drop the job and logs the reason" do
-        job.update_columns(yaml_variables: '[{"key" => "value"}]')
-
-        expect(Gitlab::ErrorTracking).to receive(:track_exception)
-                                          .with(anything, a_hash_including(build_id: job.id))
-                                          .once
-                                          .and_call_original
-
-        service.execute
-        job.reload
-
-        expect(job).to be_failed
-        expect(job).to be_data_integrity_failure
-      end
-    end
-  end
-
-  shared_examples 'job is unchanged' do
-    it 'does not change status' do
-      expect(service).to receive(:drop).exactly(3).times.and_call_original
-      expect(service).to receive(:drop_stuck).exactly(:once).and_call_original
-
-      service.execute
-      job.reload
-
-      expect(job.status).to eq(status)
-    end
-  end
-
   context 'when job is pending' do
     let(:status) { 'pending' }
 
@@ -75,13 +33,13 @@ RSpec.describe Ci::StuckBuilds::DropService do
         context 'when created_at is the same as updated_at' do
           let(:created_at) { 1.5.days.ago }
 
-          it_behaves_like 'job is dropped'
+          it_behaves_like 'job is dropped with failure reason', 'stuck_or_timeout_failure'
         end
 
         context 'when created_at is before updated_at' do
           let(:created_at) { 3.days.ago }
 
-          it_behaves_like 'job is dropped'
+          it_behaves_like 'job is dropped with failure reason', 'stuck_or_timeout_failure'
         end
 
         context 'when created_at is outside lookback window' do
@@ -149,13 +107,13 @@ RSpec.describe Ci::StuckBuilds::DropService do
         context 'when created_at is the same as updated_at' do
           let(:created_at) { 1.5.hours.ago }
 
-          it_behaves_like 'job is dropped'
+          it_behaves_like 'job is dropped with failure reason', 'stuck_or_timeout_failure'
         end
 
         context 'when created_at is before updated_at' do
           let(:created_at) { 3.days.ago }
 
-          it_behaves_like 'job is dropped'
+          it_behaves_like 'job is dropped with failure reason', 'stuck_or_timeout_failure'
         end
 
         context 'when created_at is outside lookback window' do
@@ -195,7 +153,7 @@ RSpec.describe Ci::StuckBuilds::DropService do
     context 'when job was updated_at more than an hour ago' do
       let(:updated_at) { 2.hours.ago }
 
-      it_behaves_like 'job is dropped'
+      it_behaves_like 'job is unchanged'
     end
 
     context 'when job was updated in less than 1 hour ago' do
@@ -238,47 +196,6 @@ RSpec.describe Ci::StuckBuilds::DropService do
       job.project.update!(pending_delete: true)
     end
 
-    it_behaves_like 'job is dropped'
-  end
-
-  describe 'drop stale scheduled builds' do
-    let(:status) { 'scheduled' }
-    let(:updated_at) { }
-
-    context 'when scheduled at 2 hours ago but it is not executed yet' do
-      let!(:job) { create(:ci_build, :scheduled, scheduled_at: 2.hours.ago) }
-
-      it 'drops the stale scheduled build' do
-        expect(Ci::Build.scheduled.count).to eq(1)
-        expect(job).to be_scheduled
-
-        service.execute
-        job.reload
-
-        expect(Ci::Build.scheduled.count).to eq(0)
-        expect(job).to be_failed
-        expect(job).to be_stale_schedule
-      end
-    end
-
-    context 'when scheduled at 30 minutes ago but it is not executed yet' do
-      let!(:job) { create(:ci_build, :scheduled, scheduled_at: 30.minutes.ago) }
-
-      it 'does not drop the stale scheduled build yet' do
-        expect(Ci::Build.scheduled.count).to eq(1)
-        expect(job).to be_scheduled
-
-        service.execute
-
-        expect(Ci::Build.scheduled.count).to eq(1)
-        expect(job).to be_scheduled
-      end
-    end
-
-    context 'when there are no stale scheduled builds' do
-      it 'does not drop the stale scheduled build yet' do
-        expect { service.execute }.not_to raise_error
-      end
-    end
+    it_behaves_like 'job is unchanged'
   end
 end
diff --git a/spec/services/ci/stuck_builds/drop_running_service_spec.rb b/spec/services/ci/stuck_builds/drop_running_service_spec.rb
new file mode 100644
index 00000000000..c1c92c2b8e2
--- /dev/null
+++ b/spec/services/ci/stuck_builds/drop_running_service_spec.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::StuckBuilds::DropRunningService do
+  let!(:runner) { create :ci_runner }
+  let!(:job) { create(:ci_build, runner: runner, created_at: created_at, updated_at: updated_at, status: status) }
+
+  subject(:service) { described_class.new }
+
+  around do |example|
+    freeze_time { example.run }
+  end
+
+  shared_examples 'running builds' do
+    context 'when job is running' do
+      let(:status) { 'running' }
+      let(:outdated_time) { described_class::BUILD_RUNNING_OUTDATED_TIMEOUT.ago - 30.minutes }
+      let(:fresh_time) { described_class::BUILD_RUNNING_OUTDATED_TIMEOUT.ago + 30.minutes }
+
+      context 'when job is outdated' do
+        let(:created_at) { outdated_time }
+        let(:updated_at) { outdated_time }
+
+        it_behaves_like 'job is dropped with failure reason', 'stuck_or_timeout_failure'
+      end
+
+      context 'when job is fresh' do
+        let(:created_at) { fresh_time }
+        let(:updated_at) { fresh_time }
+
+        it_behaves_like 'job is unchanged'
+      end
+
+      context 'when job freshly updated' do
+        let(:created_at) { outdated_time }
+        let(:updated_at) { fresh_time }
+
+        it_behaves_like 'job is unchanged'
+      end
+    end
+  end
+
+  include_examples 'running builds'
+
+  context 'when new query flag is disabled' do
+    before do
+      stub_feature_flags(ci_new_query_for_running_stuck_jobs: false)
+    end
+
+    include_examples 'running builds'
+  end
+
+  %w(success skipped failed canceled scheduled pending).each do |status|
+    context "when job is #{status}" do
+      let(:status) { status }
+      let(:updated_at) { 2.days.ago }
+
+      context 'when created_at is the same as updated_at' do
+        let(:created_at) { 2.days.ago }
+
+        it_behaves_like 'job is unchanged'
+      end
+
+      context 'when created_at is before updated_at' do
+        let(:created_at) { 3.days.ago }
+
+        it_behaves_like 'job is unchanged'
+      end
+    end
+  end
+end
diff --git a/spec/services/ci/stuck_builds/drop_scheduled_service_spec.rb b/spec/services/ci/stuck_builds/drop_scheduled_service_spec.rb
new file mode 100644
index 00000000000..1416fab3d25
--- /dev/null
+++ b/spec/services/ci/stuck_builds/drop_scheduled_service_spec.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::StuckBuilds::DropScheduledService do
+  let_it_be(:runner) { create :ci_runner }
+
+  let!(:job) { create :ci_build, :scheduled, scheduled_at: scheduled_at, runner: runner }
+
+  subject(:service) { described_class.new }
+
+  context 'when job is scheduled' do
+    context 'for more than an hour ago' do
+      let(:scheduled_at) { 2.hours.ago }
+
+      it_behaves_like 'job is dropped with failure reason', 'stale_schedule'
+    end
+
+    context 'for less than 1 hour ago' do
+      let(:scheduled_at) { 30.minutes.ago }
+
+      it_behaves_like 'job is unchanged'
+    end
+  end
+
+  %w(success skipped failed canceled running pending).each do |status|
+    context "when job is #{status}" do
+      before do
+        job.update!(status: status)
+      end
+
+      context 'and scheduled for more than an hour ago' do
+        let(:scheduled_at) { 2.hours.ago }
+
+        it_behaves_like 'job is unchanged'
+      end
+
+      context 'and scheduled for less than 1 hour ago' do
+        let(:scheduled_at) { 30.minutes.ago }
+
+        it_behaves_like 'job is unchanged'
+      end
+    end
+  end
+
+  context 'when there are no stale scheduled builds' do
+    let(:job) { }
+
+    it 'does not drop the stale scheduled build yet' do
+      expect { service.execute }.not_to raise_error
+    end
+  end
+end
diff --git a/spec/services/ci/update_build_state_service_spec.rb b/spec/services/ci/update_build_state_service_spec.rb
index 5bb3843da8f..e4dd3d0500f 100644
--- a/spec/services/ci/update_build_state_service_spec.rb
+++ b/spec/services/ci/update_build_state_service_spec.rb
@@ -112,6 +112,14 @@ RSpec.describe Ci::UpdateBuildStateService do
           .not_to have_received(:increment_trace_operation)
           .with(operation: :invalid)
       end
+
+      it 'does not increment chunks_invalid_checksum trace metric' do
+        execute_with_stubbed_metrics!
+
+        expect(metrics)
+          .not_to have_received(:increment_error_counter)
+          .with(type: :chunks_invalid_checksum)
+      end
     end
 
     context 'when build trace has been migrated' do
@@ -174,6 +182,14 @@ RSpec.describe Ci::UpdateBuildStateService do
             .to have_received(:increment_trace_operation)
             .with(operation: :invalid)
         end
+
+        it 'increments chunks_invalid_checksum trace metric' do
+          execute_with_stubbed_metrics!
+
+          expect(metrics)
+            .to have_received(:increment_error_counter)
+            .with(type: :chunks_invalid_checksum)
+        end
       end
 
       context 'when trace checksum is valid' do
@@ -191,6 +207,14 @@ RSpec.describe Ci::UpdateBuildStateService do
           expect(metrics)
             .not_to have_received(:increment_trace_operation)
             .with(operation: :corrupted)
+
+          expect(metrics)
+            .not_to have_received(:increment_error_counter)
+            .with(type: :chunks_invalid_checksum)
+
+          expect(metrics)
+            .not_to have_received(:increment_error_counter)
+            .with(type: :chunks_invalid_size)
         end
 
         context 'when using deprecated parameters' do
@@ -208,6 +232,14 @@ RSpec.describe Ci::UpdateBuildStateService do
             expect(metrics)
               .not_to have_received(:increment_trace_operation)
               .with(operation: :corrupted)
+
+            expect(metrics)
+              .not_to have_received(:increment_error_counter)
+              .with(type: :chunks_invalid_checksum)
+
+            expect(metrics)
+              .not_to have_received(:increment_error_counter)
+              .with(type: :chunks_invalid_size)
           end
         end
       end
@@ -227,6 +259,14 @@ RSpec.describe Ci::UpdateBuildStateService do
           expect(metrics)
             .to have_received(:increment_trace_operation)
             .with(operation: :corrupted)
+
+          expect(metrics)
+            .to have_received(:increment_error_counter)
+            .with(type: :chunks_invalid_checksum)
+
+          expect(metrics)
+            .to have_received(:increment_error_counter)
+            .with(type: :chunks_invalid_size)
         end
       end
 
@@ -243,8 +283,16 @@ RSpec.describe Ci::UpdateBuildStateService do
             .with(operation: :invalid)
 
           expect(metrics)
+            .to have_received(:increment_error_counter)
+            .with(type: :chunks_invalid_checksum)
+
+          expect(metrics)
             .not_to have_received(:increment_trace_operation)
             .with(operation: :corrupted)
+
+          expect(metrics)
+            .not_to have_received(:increment_error_counter)
+            .with(type: :chunks_invalid_size)
         end
       end
 
@@ -325,6 +373,10 @@ RSpec.describe Ci::UpdateBuildStateService do
         expect(metrics)
           .not_to have_received(:increment_trace_operation)
           .with(operation: :invalid)
+
+        expect(metrics)
+          .not_to have_received(:increment_error_counter)
+          .with(type: :chunks_invalid_checksum)
       end
 
       context 'when build pending state is outdated' do
diff --git a/spec/services/ci/update_pending_build_service_spec.rb b/spec/services/ci/update_pending_build_service_spec.rb
index d842042de40..d36564938c8 100644
--- a/spec/services/ci/update_pending_build_service_spec.rb
+++ b/spec/services/ci/update_pending_build_service_spec.rb
@@ -3,21 +3,23 @@
 require 'spec_helper'
 
 RSpec.describe Ci::UpdatePendingBuildService do
-  describe '#execute' do
-    let_it_be(:group) { create(:group) }
-    let_it_be(:project) { create(:project, namespace: group) }
-    let_it_be(:pending_build_1) { create(:ci_pending_build, project: project, instance_runners_enabled: false) }
-    let_it_be(:pending_build_2) { create(:ci_pending_build, project: project, instance_runners_enabled: true) }
-    let_it_be(:update_params) { { instance_runners_enabled: true } }
+  let_it_be(:group) { create(:group) }
+  let_it_be(:project) { create(:project, namespace: group) }
+  let_it_be_with_reload(:pending_build_1) { create(:ci_pending_build, project: project, instance_runners_enabled: false) }
+  let_it_be_with_reload(:pending_build_2) { create(:ci_pending_build, project: project, instance_runners_enabled: true) }
+  let_it_be(:update_params) { { instance_runners_enabled: true } }
+
+  let(:service) { described_class.new(model, update_params) }
 
-    subject(:service) { described_class.new(model, update_params).execute }
+  describe '#execute' do
+    subject(:update_pending_builds) { service.execute }
 
     context 'validations' do
       context 'when model is invalid' do
         let(:model) { pending_build_1 }
 
         it 'raises an error' do
-          expect { service }.to raise_error(described_class::InvalidModelError)
+          expect { update_pending_builds }.to raise_error(described_class::InvalidModelError)
         end
       end
 
@@ -26,7 +28,7 @@ RSpec.describe Ci::UpdatePendingBuildService do
         let(:update_params) { { minutes_exceeded: true } }
 
         it 'raises an error' do
-          expect { service }.to raise_error(described_class::InvalidParamsError)
+          expect { update_pending_builds }.to raise_error(described_class::InvalidParamsError)
         end
       end
     end
@@ -35,10 +37,10 @@ RSpec.describe Ci::UpdatePendingBuildService do
       let(:model) { group }
 
       it 'updates all pending builds', :aggregate_failures do
-        service
+        update_pending_builds
 
-        expect(pending_build_1.reload.instance_runners_enabled).to be_truthy
-        expect(pending_build_2.reload.instance_runners_enabled).to be_truthy
+        expect(pending_build_1.instance_runners_enabled).to be_truthy
+        expect(pending_build_2.instance_runners_enabled).to be_truthy
       end
 
       context 'when ci_pending_builds_maintain_shared_runners_data is disabled' do
@@ -47,10 +49,10 @@ RSpec.describe Ci::UpdatePendingBuildService do
         end
 
         it 'does not update all pending builds', :aggregate_failures do
-          service
+          update_pending_builds
 
-          expect(pending_build_1.reload.instance_runners_enabled).to be_falsey
-          expect(pending_build_2.reload.instance_runners_enabled).to be_truthy
+          expect(pending_build_1.instance_runners_enabled).to be_falsey
+          expect(pending_build_2.instance_runners_enabled).to be_truthy
         end
       end
     end
@@ -59,10 +61,10 @@ RSpec.describe Ci::UpdatePendingBuildService do
       let(:model) { project }
 
       it 'updates all pending builds', :aggregate_failures do
-        service
+        update_pending_builds
 
-        expect(pending_build_1.reload.instance_runners_enabled).to be_truthy
-        expect(pending_build_2.reload.instance_runners_enabled).to be_truthy
+        expect(pending_build_1.instance_runners_enabled).to be_truthy
+        expect(pending_build_2.instance_runners_enabled).to be_truthy
       end
 
       context 'when ci_pending_builds_maintain_shared_runners_data is disabled' do
@@ -71,10 +73,10 @@ RSpec.describe Ci::UpdatePendingBuildService do
         end
 
         it 'does not update all pending builds', :aggregate_failures do
-          service
+          update_pending_builds
 
-          expect(pending_build_1.reload.instance_runners_enabled).to be_falsey
-          expect(pending_build_2.reload.instance_runners_enabled).to be_truthy
+          expect(pending_build_1.instance_runners_enabled).to be_falsey
+          expect(pending_build_2.instance_runners_enabled).to be_truthy
         end
       end
     end