From d9ab72d6080f594d0b3cae15f14b3ef2c6c638cb Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 20 Oct 2021 08:43:02 +0000 Subject: Add latest changes from gitlab-org/gitlab@14-4-stable-ee --- spec/services/ci/archive_trace_service_spec.rb | 26 ++ .../ci/create_pipeline_service/include_spec.rb | 22 -- spec/services/ci/drop_pipeline_service_spec.rb | 3 +- spec/services/ci/pipelines/add_job_service_spec.rb | 13 - spec/services/ci/pipelines/hook_service_spec.rb | 47 ++++ spec/services/ci/play_bridge_service_spec.rb | 12 +- spec/services/ci/process_pipeline_service_spec.rb | 68 +---- spec/services/ci/register_job_service_spec.rb | 50 +++- ...gn_resource_from_resource_group_service_spec.rb | 86 +++++++ spec/services/ci/retry_build_service_spec.rb | 13 +- spec/services/ci/retry_pipeline_service_spec.rb | 39 ++- .../ci/stuck_builds/drop_pending_service_spec.rb | 201 +++++++++++++++ .../ci/stuck_builds/drop_running_service_spec.rb | 72 ++++++ .../ci/stuck_builds/drop_scheduled_service_spec.rb | 53 ++++ spec/services/ci/stuck_builds/drop_service_spec.rb | 284 --------------------- .../services/ci/update_build_state_service_spec.rb | 52 ++++ .../ci/update_pending_build_service_spec.rb | 44 ++-- 17 files changed, 655 insertions(+), 430 deletions(-) create mode 100644 spec/services/ci/pipelines/hook_service_spec.rb create mode 100644 spec/services/ci/stuck_builds/drop_pending_service_spec.rb create mode 100644 spec/services/ci/stuck_builds/drop_running_service_spec.rb create mode 100644 spec/services/ci/stuck_builds/drop_scheduled_service_spec.rb delete mode 100644 spec/services/ci/stuck_builds/drop_service_spec.rb (limited to 'spec/services/ci') diff --git a/spec/services/ci/archive_trace_service_spec.rb b/spec/services/ci/archive_trace_service_spec.rb index 071b5c3b2f9..b08ba6fd5e5 100644 --- a/spec/services/ci/archive_trace_service_spec.rb +++ b/spec/services/ci/archive_trace_service_spec.rb @@ -88,6 +88,32 @@ RSpec.describe Ci::ArchiveTraceService, '#execute' do subject end + + context 'job has archive and chunks' do + let(:job) { create(:ci_build, :success, :trace_artifact) } + + before do + create(:ci_build_trace_chunk, build: job, chunk_index: 0) + end + + context 'archive is not completed' do + before do + job.job_artifacts_trace.file.remove! + end + + it 'cleanups any stale archive data' do + expect(job.job_artifacts_trace).to be_present + + subject + + expect(job.reload.job_artifacts_trace).to be_nil + end + end + + it 'removes trace chunks' do + expect { subject }.to change { job.trace_chunks.count }.to(0) + end + end end context 'when the archival process is backed off' do diff --git a/spec/services/ci/create_pipeline_service/include_spec.rb b/spec/services/ci/create_pipeline_service/include_spec.rb index 46271ee36c0..5e7dace8e15 100644 --- a/spec/services/ci/create_pipeline_service/include_spec.rb +++ b/spec/services/ci/create_pipeline_service/include_spec.rb @@ -58,17 +58,6 @@ RSpec.describe Ci::CreatePipelineService do expect(pipeline).to be_created_successfully expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec') end - - context 'when the FF ci_include_rules is disabled' do - before do - stub_feature_flags(ci_include_rules: false) - end - - it 'includes the job in the file' do - expect(pipeline).to be_created_successfully - expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec') - end - end end context 'when the rules does not match' do @@ -78,17 +67,6 @@ RSpec.describe Ci::CreatePipelineService do expect(pipeline).to be_created_successfully expect(pipeline.processables.pluck(:name)).to contain_exactly('job') end - - context 'when the FF ci_include_rules is disabled' do - before do - stub_feature_flags(ci_include_rules: false) - end - - it 'includes the job in the file' do - expect(pipeline).to be_created_successfully - expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec') - end - end end end end diff --git a/spec/services/ci/drop_pipeline_service_spec.rb b/spec/services/ci/drop_pipeline_service_spec.rb index c6a118c6083..ddb53712d9c 100644 --- a/spec/services/ci/drop_pipeline_service_spec.rb +++ b/spec/services/ci/drop_pipeline_service_spec.rb @@ -50,13 +50,14 @@ RSpec.describe Ci::DropPipelineService do end.count writes_per_build = 2 + load_balancer_queries = 3 expected_reads_count = control_count - writes_per_build create_list(:ci_build, 5, :running, pipeline: cancelable_pipeline) expect do drop_pipeline!(cancelable_pipeline) - end.not_to exceed_query_limit(expected_reads_count + (5 * writes_per_build)) + end.not_to exceed_query_limit(expected_reads_count + (5 * writes_per_build) + load_balancer_queries) end end end diff --git a/spec/services/ci/pipelines/add_job_service_spec.rb b/spec/services/ci/pipelines/add_job_service_spec.rb index 3a77d26dd9e..709a840c644 100644 --- a/spec/services/ci/pipelines/add_job_service_spec.rb +++ b/spec/services/ci/pipelines/add_job_service_spec.rb @@ -77,19 +77,6 @@ RSpec.describe Ci::Pipelines::AddJobService do expect(execute).to be_success expect(execute.payload[:job]).to eq(job) end - - context 'when the FF ci_pipeline_add_job_with_lock is disabled' do - before do - stub_feature_flags(ci_pipeline_add_job_with_lock: false) - end - - it 'does not use exclusive lock' do - expect(Gitlab::ExclusiveLease).not_to receive(:new).with(lock_key, timeout: lock_timeout) - - expect(execute).to be_success - expect(execute.payload[:job]).to eq(job) - end - end end end end diff --git a/spec/services/ci/pipelines/hook_service_spec.rb b/spec/services/ci/pipelines/hook_service_spec.rb new file mode 100644 index 00000000000..0e1ef6afd0d --- /dev/null +++ b/spec/services/ci/pipelines/hook_service_spec.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::Pipelines::HookService do + describe '#execute_hooks' do + let_it_be(:namespace) { create(:namespace) } + let_it_be(:project) { create(:project, :repository, namespace: namespace) } + let_it_be(:pipeline) { create(:ci_empty_pipeline, :created, project: project) } + + let(:hook_enabled) { true } + let!(:hook) { create(:project_hook, project: project, pipeline_events: hook_enabled) } + let(:hook_data) { double } + + subject(:service) { described_class.new(pipeline) } + + describe 'HOOK_NAME' do + specify { expect(described_class::HOOK_NAME).to eq(:pipeline_hooks) } + end + + context 'with pipeline hooks enabled' do + before do + allow(Gitlab::DataBuilder::Pipeline).to receive(:build).with(pipeline).once.and_return(hook_data) + end + + it 'calls pipeline.project.execute_hooks and pipeline.project.execute_integrations' do + create(:pipelines_email_integration, project: project) + + expect(pipeline.project).to receive(:execute_hooks).with(hook_data, described_class::HOOK_NAME) + expect(pipeline.project).to receive(:execute_integrations).with(hook_data, described_class::HOOK_NAME) + + service.execute + end + end + + context 'with pipeline hooks and integrations disabled' do + let(:hook_enabled) { false } + + it 'does not call pipeline.project.execute_hooks and pipeline.project.execute_integrations' do + expect(pipeline.project).not_to receive(:execute_hooks) + expect(pipeline.project).not_to receive(:execute_integrations) + + service.execute + end + end + end +end diff --git a/spec/services/ci/play_bridge_service_spec.rb b/spec/services/ci/play_bridge_service_spec.rb index 3f97bfdf5ae..56b1615a56d 100644 --- a/spec/services/ci/play_bridge_service_spec.rb +++ b/spec/services/ci/play_bridge_service_spec.rb @@ -23,18 +23,18 @@ RSpec.describe Ci::PlayBridgeService, '#execute' do expect(bridge.reload).to be_pending end - it 'enqueues Ci::CreateCrossProjectPipelineWorker' do - expect(::Ci::CreateCrossProjectPipelineWorker).to receive(:perform_async).with(bridge.id) - - execute_service - end - it "updates bridge's user" do execute_service expect(bridge.reload.user).to eq(user) end + it 'enqueues Ci::CreateDownstreamPipelineWorker' do + expect(::Ci::CreateDownstreamPipelineWorker).to receive(:perform_async).with(bridge.id) + + execute_service + end + context 'when a subsequent job is skipped' do let!(:job) { create(:ci_build, :skipped, pipeline: pipeline, stage_idx: bridge.stage_idx + 1) } diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb index b5bf0adadaf..404e1bf7c87 100644 --- a/spec/services/ci/process_pipeline_service_spec.rb +++ b/spec/services/ci/process_pipeline_service_spec.rb @@ -10,11 +10,9 @@ RSpec.describe Ci::ProcessPipelineService do end let(:pipeline_processing_events_counter) { double(increment: true) } - let(:legacy_update_jobs_counter) { double(increment: true) } let(:metrics) do - double(pipeline_processing_events_counter: pipeline_processing_events_counter, - legacy_update_jobs_counter: legacy_update_jobs_counter) + double(pipeline_processing_events_counter: pipeline_processing_events_counter) end subject { described_class.new(pipeline) } @@ -33,68 +31,4 @@ RSpec.describe Ci::ProcessPipelineService do subject.execute end end - - describe 'updating a list of retried builds' do - let!(:build_retried) { create_build('build') } - let!(:build) { create_build('build') } - let!(:test) { create_build('test') } - - context 'when FF ci_remove_update_retried_from_process_pipeline is enabled' do - it 'does not update older builds as retried' do - subject.execute - - expect(all_builds.latest).to contain_exactly(build, build_retried, test) - expect(all_builds.retried).to be_empty - end - end - - context 'when FF ci_remove_update_retried_from_process_pipeline is disabled' do - before do - stub_feature_flags(ci_remove_update_retried_from_process_pipeline: false) - end - - it 'returns unique statuses' do - subject.execute - - expect(all_builds.latest).to contain_exactly(build, test) - expect(all_builds.retried).to contain_exactly(build_retried) - end - - it 'increments the counter' do - expect(legacy_update_jobs_counter).to receive(:increment) - - subject.execute - end - - it 'logs the project and pipeline id' do - expect(Gitlab::AppJsonLogger).to receive(:info).with(event: 'update_retried_is_used', - project_id: project.id, - pipeline_id: pipeline.id) - - subject.execute - end - - context 'when the previous build has already retried column true' do - before do - build_retried.update_columns(retried: true) - end - - it 'does not increment the counter' do - expect(legacy_update_jobs_counter).not_to receive(:increment) - - subject.execute - end - end - end - - private - - def create_build(name, **opts) - create(:ci_build, :created, pipeline: pipeline, name: name, **opts) - end - - def all_builds - pipeline.builds.order(:stage_idx, :id) - end - end end diff --git a/spec/services/ci/register_job_service_spec.rb b/spec/services/ci/register_job_service_spec.rb index 73ff15ec393..650353eb751 100644 --- a/spec/services/ci/register_job_service_spec.rb +++ b/spec/services/ci/register_job_service_spec.rb @@ -14,7 +14,7 @@ module Ci let!(:pending_job) { create(:ci_build, :pending, :queued, pipeline: pipeline) } describe '#execute' do - context 'checks database loadbalancing stickiness', :db_load_balancing do + context 'checks database loadbalancing stickiness' do subject { described_class.new(shared_runner).execute } before do @@ -22,14 +22,14 @@ module Ci end it 'result is valid if replica did caught-up' do - expect(Gitlab::Database::LoadBalancing::Sticking).to receive(:all_caught_up?) + expect(ApplicationRecord.sticking).to receive(:all_caught_up?) .with(:runner, shared_runner.id) { true } expect(subject).to be_valid end it 'result is invalid if replica did not caught-up' do - expect(Gitlab::Database::LoadBalancing::Sticking).to receive(:all_caught_up?) + expect(ApplicationRecord.sticking).to receive(:all_caught_up?) .with(:runner, shared_runner.id) { false } expect(subject).not_to be_valid @@ -87,19 +87,25 @@ module Ci end context 'for specific runner' do - context 'with FF disabled' do + context 'with tables decoupling disabled' do before do stub_feature_flags( ci_pending_builds_project_runners_decoupling: false, ci_queueing_builds_enabled_checks: false) end + around do |example| + allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332952') do + example.run + end + end + it 'does not pick a build' do expect(execute(specific_runner)).to be_nil end end - context 'with FF enabled' do + context 'with tables decoupling enabled' do before do stub_feature_flags( ci_pending_builds_project_runners_decoupling: true, @@ -266,17 +272,23 @@ module Ci context 'and uses project runner' do let(:build) { execute(specific_runner) } - context 'with FF disabled' do + context 'with tables decoupling disabled' do before do stub_feature_flags( ci_pending_builds_project_runners_decoupling: false, ci_queueing_builds_enabled_checks: false) end + around do |example| + allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332952') do + example.run + end + end + it { expect(build).to be_nil } end - context 'with FF enabled' do + context 'with tables decoupling enabled' do before do stub_feature_flags( ci_pending_builds_project_runners_decoupling: true, @@ -791,6 +803,12 @@ module Ci stub_feature_flags(ci_queueing_denormalize_shared_runners_information: false) end + around do |example| + allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332952') do + example.run + end + end + include_examples 'handles runner assignment' end @@ -807,6 +825,12 @@ module Ci stub_feature_flags(ci_queueing_denormalize_tags_information: false) end + around do |example| + allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332952') do + example.run + end + end + include_examples 'handles runner assignment' end @@ -815,6 +839,12 @@ module Ci stub_feature_flags(ci_queueing_denormalize_namespace_traversal_ids: false) end + around do |example| + allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332952') do + example.run + end + end + include_examples 'handles runner assignment' end end @@ -824,6 +854,12 @@ module Ci stub_feature_flags(ci_pending_builds_queue_source: false) end + around do |example| + allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332952') do + example.run + end + end + include_examples 'handles runner assignment' end end diff --git a/spec/services/ci/resource_groups/assign_resource_from_resource_group_service_spec.rb b/spec/services/ci/resource_groups/assign_resource_from_resource_group_service_spec.rb index 53aa842bc28..194203a422c 100644 --- a/spec/services/ci/resource_groups/assign_resource_from_resource_group_service_spec.rb +++ b/spec/services/ci/resource_groups/assign_resource_from_resource_group_service_spec.rb @@ -48,6 +48,92 @@ RSpec.describe Ci::ResourceGroups::AssignResourceFromResourceGroupService do expect(build).to be_pending end end + + context 'when process mode is oldest_first' do + let(:resource_group) { create(:ci_resource_group, process_mode: :oldest_first, project: project) } + + it 'requests resource' do + subject + + expect(build.reload).to be_pending + expect(build.resource).to be_present + end + + context 'when the other job exists in the newer pipeline' do + let!(:build_2) { create(:ci_build, :waiting_for_resource, project: project, user: user, resource_group: resource_group) } + + it 'requests resource for the job in the oldest pipeline' do + subject + + expect(build.reload).to be_pending + expect(build.resource).to be_present + expect(build_2.reload).to be_waiting_for_resource + expect(build_2.resource).to be_nil + end + end + + context 'when build is not `waiting_for_resource` state' do + let!(:build) { create(:ci_build, :created, project: project, user: user, resource_group: resource_group) } + + it 'attempts to request a resource' do + expect_next_found_instance_of(Ci::Build) do |job| + expect(job).to receive(:enqueue_waiting_for_resource).and_call_original + end + + subject + end + + it 'does not change the job status' do + subject + + expect(build.reload).to be_created + expect(build.resource).to be_nil + end + end + end + + context 'when process mode is newest_first' do + let(:resource_group) { create(:ci_resource_group, process_mode: :newest_first, project: project) } + + it 'requests resource' do + subject + + expect(build.reload).to be_pending + expect(build.resource).to be_present + end + + context 'when the other job exists in the newer pipeline' do + let!(:build_2) { create(:ci_build, :waiting_for_resource, project: project, user: user, resource_group: resource_group) } + + it 'requests resource for the job in the newest pipeline' do + subject + + expect(build.reload).to be_waiting_for_resource + expect(build.resource).to be_nil + expect(build_2.reload).to be_pending + expect(build_2.resource).to be_present + end + end + + context 'when build is not `waiting_for_resource` state' do + let!(:build) { create(:ci_build, :created, project: project, user: user, resource_group: resource_group) } + + it 'attempts to request a resource' do + expect_next_found_instance_of(Ci::Build) do |job| + expect(job).to receive(:enqueue_waiting_for_resource).and_call_original + end + + subject + end + + it 'does not change the job status' do + subject + + expect(build.reload).to be_created + expect(build.resource).to be_nil + end + end + end end context 'when there are no available resources' do diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb index ce2e6ba5e15..15c88c9f657 100644 --- a/spec/services/ci/retry_build_service_spec.rb +++ b/spec/services/ci/retry_build_service_spec.rb @@ -48,7 +48,7 @@ RSpec.describe Ci::RetryBuildService do job_artifacts_network_referee job_artifacts_dotenv job_artifacts_cobertura needs job_artifacts_accessibility job_artifacts_requirements job_artifacts_coverage_fuzzing - job_artifacts_api_fuzzing].freeze + job_artifacts_api_fuzzing terraform_state_versions].freeze ignore_accessors = %i[type lock_version target_url base_tags trace_sections @@ -88,6 +88,7 @@ RSpec.describe Ci::RetryBuildService do create(:ci_job_variable, job: build) create(:ci_build_need, build: build) + create(:terraform_state_version, build: build) end describe 'clone accessors' do @@ -276,13 +277,17 @@ RSpec.describe Ci::RetryBuildService do end end - describe '#reprocess' do + describe '#clone!' do let(:new_build) do travel_to(1.second.from_now) do - service.reprocess!(build) + service.clone!(build) end end + it 'raises an error when an unexpected class is passed' do + expect { service.clone!(create(:ci_build).present) }.to raise_error(TypeError) + end + context 'when user has ability to execute build' do before do stub_not_protect_default_branch @@ -338,7 +343,7 @@ RSpec.describe Ci::RetryBuildService do let(:user) { reporter } it 'raises an error' do - expect { service.reprocess!(build) } + expect { service.clone!(build) } .to raise_error Gitlab::Access::AccessDeniedError end end diff --git a/spec/services/ci/retry_pipeline_service_spec.rb b/spec/services/ci/retry_pipeline_service_spec.rb index 3e2e9f07723..12106b70969 100644 --- a/spec/services/ci/retry_pipeline_service_spec.rb +++ b/spec/services/ci/retry_pipeline_service_spec.rb @@ -316,6 +316,26 @@ RSpec.describe Ci::RetryPipelineService, '#execute' do expect(bridge.reload).to be_pending end end + + context 'when there are skipped jobs in later stages' do + before do + create_build('build 1', :success, 0) + create_build('test 2', :failed, 1) + create_build('report 3', :skipped, 2) + create_bridge('deploy 4', :skipped, 2) + end + + it 'retries failed jobs and processes skipped jobs' do + service.execute(pipeline) + + expect(build('build 1')).to be_success + expect(build('test 2')).to be_pending + expect(build('report 3')).to be_created + expect(build('deploy 4')).to be_created + + expect(pipeline.reload).to be_running + end + end end context 'when user is not allowed to retry pipeline' do @@ -390,16 +410,25 @@ RSpec.describe Ci::RetryPipelineService, '#execute' do pipeline.reload.statuses end + # The method name can be confusing because this can actually return both Ci::Build and Ci::Bridge def build(name) statuses.latest.find_by(name: name) end def create_build(name, status, stage_num, **opts) - create(:ci_build, name: name, - status: status, - stage: "stage_#{stage_num}", - stage_idx: stage_num, - pipeline: pipeline, **opts) do |build| + create_processable(:ci_build, name, status, stage_num, **opts) + end + + def create_bridge(name, status, stage_num, **opts) + create_processable(:ci_bridge, name, status, stage_num, **opts) + end + + def create_processable(type, name, status, stage_num, **opts) + create(type, name: name, + status: status, + stage: "stage_#{stage_num}", + stage_idx: stage_num, + pipeline: pipeline, **opts) do |_job| ::Ci::ProcessPipelineService.new(pipeline).execute end end diff --git a/spec/services/ci/stuck_builds/drop_pending_service_spec.rb b/spec/services/ci/stuck_builds/drop_pending_service_spec.rb new file mode 100644 index 00000000000..aa0526edf57 --- /dev/null +++ b/spec/services/ci/stuck_builds/drop_pending_service_spec.rb @@ -0,0 +1,201 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::StuckBuilds::DropPendingService do + let!(:runner) { create :ci_runner } + let!(:job) { create :ci_build, runner: runner } + let(:created_at) { } + let(:updated_at) { } + + subject(:service) { described_class.new } + + before do + job_attributes = { status: status } + job_attributes[:created_at] = created_at if created_at + job_attributes[:updated_at] = updated_at if updated_at + job.update!(job_attributes) + end + + context 'when job is pending' do + let(:status) { 'pending' } + + context 'when job is not stuck' do + before do + allow_next_found_instance_of(Ci::Build) do |build| + allow(build).to receive(:stuck?).and_return(false) + end + end + + context 'when job was updated_at more than 1 day ago' do + let(:updated_at) { 1.5.days.ago } + + context 'when created_at is the same as updated_at' do + let(:created_at) { 1.5.days.ago } + + it_behaves_like 'job is dropped with failure reason', 'stuck_or_timeout_failure' + end + + context 'when created_at is before updated_at' do + let(:created_at) { 3.days.ago } + + it_behaves_like 'job is dropped with failure reason', 'stuck_or_timeout_failure' + end + + context 'when created_at is outside lookback window' do + let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } + + it_behaves_like 'job is unchanged' + end + end + + context 'when job was updated less than 1 day ago' do + let(:updated_at) { 6.hours.ago } + + context 'when created_at is the same as updated_at' do + let(:created_at) { 1.5.days.ago } + + it_behaves_like 'job is unchanged' + end + + context 'when created_at is before updated_at' do + let(:created_at) { 3.days.ago } + + it_behaves_like 'job is unchanged' + end + + context 'when created_at is outside lookback window' do + let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } + + it_behaves_like 'job is unchanged' + end + end + + context 'when job was updated more than 1 hour ago' do + let(:updated_at) { 2.hours.ago } + + context 'when created_at is the same as updated_at' do + let(:created_at) { 2.hours.ago } + + it_behaves_like 'job is unchanged' + end + + context 'when created_at is before updated_at' do + let(:created_at) { 3.days.ago } + + it_behaves_like 'job is unchanged' + end + + context 'when created_at is outside lookback window' do + let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } + + it_behaves_like 'job is unchanged' + end + end + end + + context 'when job is stuck' do + before do + allow_next_found_instance_of(Ci::Build) do |build| + allow(build).to receive(:stuck?).and_return(true) + end + end + + context 'when job was updated_at more than 1 hour ago' do + let(:updated_at) { 1.5.hours.ago } + + context 'when created_at is the same as updated_at' do + let(:created_at) { 1.5.hours.ago } + + it_behaves_like 'job is dropped with failure reason', 'stuck_or_timeout_failure' + end + + context 'when created_at is before updated_at' do + let(:created_at) { 3.days.ago } + + it_behaves_like 'job is dropped with failure reason', 'stuck_or_timeout_failure' + end + + context 'when created_at is outside lookback window' do + let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } + + it_behaves_like 'job is unchanged' + end + end + + context 'when job was updated in less than 1 hour ago' do + let(:updated_at) { 30.minutes.ago } + + context 'when created_at is the same as updated_at' do + let(:created_at) { 30.minutes.ago } + + it_behaves_like 'job is unchanged' + end + + context 'when created_at is before updated_at' do + let(:created_at) { 2.days.ago } + + it_behaves_like 'job is unchanged' + end + + context 'when created_at is outside lookback window' do + let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } + + it_behaves_like 'job is unchanged' + end + end + end + end + + context 'when job is running' do + let(:status) { 'running' } + + context 'when job was updated_at more than an hour ago' do + let(:updated_at) { 2.hours.ago } + + it_behaves_like 'job is unchanged' + end + + context 'when job was updated in less than 1 hour ago' do + let(:updated_at) { 30.minutes.ago } + + it_behaves_like 'job is unchanged' + end + end + + %w(success skipped failed canceled).each do |status| + context "when job is #{status}" do + let(:status) { status } + let(:updated_at) { 2.days.ago } + + context 'when created_at is the same as updated_at' do + let(:created_at) { 2.days.ago } + + it_behaves_like 'job is unchanged' + end + + context 'when created_at is before updated_at' do + let(:created_at) { 3.days.ago } + + it_behaves_like 'job is unchanged' + end + + context 'when created_at is outside lookback window' do + let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } + + it_behaves_like 'job is unchanged' + end + end + end + + context 'for deleted project' do + let(:status) { 'running' } + let(:updated_at) { 2.days.ago } + + before do + job.project.update!(pending_delete: true) + end + + it_behaves_like 'job is unchanged' + end +end diff --git a/spec/services/ci/stuck_builds/drop_running_service_spec.rb b/spec/services/ci/stuck_builds/drop_running_service_spec.rb new file mode 100644 index 00000000000..c1c92c2b8e2 --- /dev/null +++ b/spec/services/ci/stuck_builds/drop_running_service_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::StuckBuilds::DropRunningService do + let!(:runner) { create :ci_runner } + let!(:job) { create(:ci_build, runner: runner, created_at: created_at, updated_at: updated_at, status: status) } + + subject(:service) { described_class.new } + + around do |example| + freeze_time { example.run } + end + + shared_examples 'running builds' do + context 'when job is running' do + let(:status) { 'running' } + let(:outdated_time) { described_class::BUILD_RUNNING_OUTDATED_TIMEOUT.ago - 30.minutes } + let(:fresh_time) { described_class::BUILD_RUNNING_OUTDATED_TIMEOUT.ago + 30.minutes } + + context 'when job is outdated' do + let(:created_at) { outdated_time } + let(:updated_at) { outdated_time } + + it_behaves_like 'job is dropped with failure reason', 'stuck_or_timeout_failure' + end + + context 'when job is fresh' do + let(:created_at) { fresh_time } + let(:updated_at) { fresh_time } + + it_behaves_like 'job is unchanged' + end + + context 'when job freshly updated' do + let(:created_at) { outdated_time } + let(:updated_at) { fresh_time } + + it_behaves_like 'job is unchanged' + end + end + end + + include_examples 'running builds' + + context 'when new query flag is disabled' do + before do + stub_feature_flags(ci_new_query_for_running_stuck_jobs: false) + end + + include_examples 'running builds' + end + + %w(success skipped failed canceled scheduled pending).each do |status| + context "when job is #{status}" do + let(:status) { status } + let(:updated_at) { 2.days.ago } + + context 'when created_at is the same as updated_at' do + let(:created_at) { 2.days.ago } + + it_behaves_like 'job is unchanged' + end + + context 'when created_at is before updated_at' do + let(:created_at) { 3.days.ago } + + it_behaves_like 'job is unchanged' + end + end + end +end diff --git a/spec/services/ci/stuck_builds/drop_scheduled_service_spec.rb b/spec/services/ci/stuck_builds/drop_scheduled_service_spec.rb new file mode 100644 index 00000000000..1416fab3d25 --- /dev/null +++ b/spec/services/ci/stuck_builds/drop_scheduled_service_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::StuckBuilds::DropScheduledService do + let_it_be(:runner) { create :ci_runner } + + let!(:job) { create :ci_build, :scheduled, scheduled_at: scheduled_at, runner: runner } + + subject(:service) { described_class.new } + + context 'when job is scheduled' do + context 'for more than an hour ago' do + let(:scheduled_at) { 2.hours.ago } + + it_behaves_like 'job is dropped with failure reason', 'stale_schedule' + end + + context 'for less than 1 hour ago' do + let(:scheduled_at) { 30.minutes.ago } + + it_behaves_like 'job is unchanged' + end + end + + %w(success skipped failed canceled running pending).each do |status| + context "when job is #{status}" do + before do + job.update!(status: status) + end + + context 'and scheduled for more than an hour ago' do + let(:scheduled_at) { 2.hours.ago } + + it_behaves_like 'job is unchanged' + end + + context 'and scheduled for less than 1 hour ago' do + let(:scheduled_at) { 30.minutes.ago } + + it_behaves_like 'job is unchanged' + end + end + end + + context 'when there are no stale scheduled builds' do + let(:job) { } + + it 'does not drop the stale scheduled build yet' do + expect { service.execute }.not_to raise_error + end + end +end diff --git a/spec/services/ci/stuck_builds/drop_service_spec.rb b/spec/services/ci/stuck_builds/drop_service_spec.rb deleted file mode 100644 index 8dfd1bc1b3d..00000000000 --- a/spec/services/ci/stuck_builds/drop_service_spec.rb +++ /dev/null @@ -1,284 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Ci::StuckBuilds::DropService do - let!(:runner) { create :ci_runner } - let!(:job) { create :ci_build, runner: runner } - let(:created_at) { } - let(:updated_at) { } - - subject(:service) { described_class.new } - - before do - job_attributes = { status: status } - job_attributes[:created_at] = created_at if created_at - job_attributes[:updated_at] = updated_at if updated_at - job.update!(job_attributes) - end - - shared_examples 'job is dropped' do - it 'changes status' do - expect(service).to receive(:drop).exactly(3).times.and_call_original - expect(service).to receive(:drop_stuck).exactly(:once).and_call_original - - service.execute - job.reload - - expect(job).to be_failed - expect(job).to be_stuck_or_timeout_failure - end - - context 'when job have data integrity problem' do - it "does drop the job and logs the reason" do - job.update_columns(yaml_variables: '[{"key" => "value"}]') - - expect(Gitlab::ErrorTracking).to receive(:track_exception) - .with(anything, a_hash_including(build_id: job.id)) - .once - .and_call_original - - service.execute - job.reload - - expect(job).to be_failed - expect(job).to be_data_integrity_failure - end - end - end - - shared_examples 'job is unchanged' do - it 'does not change status' do - expect(service).to receive(:drop).exactly(3).times.and_call_original - expect(service).to receive(:drop_stuck).exactly(:once).and_call_original - - service.execute - job.reload - - expect(job.status).to eq(status) - end - end - - context 'when job is pending' do - let(:status) { 'pending' } - - context 'when job is not stuck' do - before do - allow_next_found_instance_of(Ci::Build) do |build| - allow(build).to receive(:stuck?).and_return(false) - end - end - - context 'when job was updated_at more than 1 day ago' do - let(:updated_at) { 1.5.days.ago } - - context 'when created_at is the same as updated_at' do - let(:created_at) { 1.5.days.ago } - - it_behaves_like 'job is dropped' - end - - context 'when created_at is before updated_at' do - let(:created_at) { 3.days.ago } - - it_behaves_like 'job is dropped' - end - - context 'when created_at is outside lookback window' do - let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } - - it_behaves_like 'job is unchanged' - end - end - - context 'when job was updated less than 1 day ago' do - let(:updated_at) { 6.hours.ago } - - context 'when created_at is the same as updated_at' do - let(:created_at) { 1.5.days.ago } - - it_behaves_like 'job is unchanged' - end - - context 'when created_at is before updated_at' do - let(:created_at) { 3.days.ago } - - it_behaves_like 'job is unchanged' - end - - context 'when created_at is outside lookback window' do - let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } - - it_behaves_like 'job is unchanged' - end - end - - context 'when job was updated more than 1 hour ago' do - let(:updated_at) { 2.hours.ago } - - context 'when created_at is the same as updated_at' do - let(:created_at) { 2.hours.ago } - - it_behaves_like 'job is unchanged' - end - - context 'when created_at is before updated_at' do - let(:created_at) { 3.days.ago } - - it_behaves_like 'job is unchanged' - end - - context 'when created_at is outside lookback window' do - let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } - - it_behaves_like 'job is unchanged' - end - end - end - - context 'when job is stuck' do - before do - allow_next_found_instance_of(Ci::Build) do |build| - allow(build).to receive(:stuck?).and_return(true) - end - end - - context 'when job was updated_at more than 1 hour ago' do - let(:updated_at) { 1.5.hours.ago } - - context 'when created_at is the same as updated_at' do - let(:created_at) { 1.5.hours.ago } - - it_behaves_like 'job is dropped' - end - - context 'when created_at is before updated_at' do - let(:created_at) { 3.days.ago } - - it_behaves_like 'job is dropped' - end - - context 'when created_at is outside lookback window' do - let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } - - it_behaves_like 'job is unchanged' - end - end - - context 'when job was updated in less than 1 hour ago' do - let(:updated_at) { 30.minutes.ago } - - context 'when created_at is the same as updated_at' do - let(:created_at) { 30.minutes.ago } - - it_behaves_like 'job is unchanged' - end - - context 'when created_at is before updated_at' do - let(:created_at) { 2.days.ago } - - it_behaves_like 'job is unchanged' - end - - context 'when created_at is outside lookback window' do - let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } - - it_behaves_like 'job is unchanged' - end - end - end - end - - context 'when job is running' do - let(:status) { 'running' } - - context 'when job was updated_at more than an hour ago' do - let(:updated_at) { 2.hours.ago } - - it_behaves_like 'job is dropped' - end - - context 'when job was updated in less than 1 hour ago' do - let(:updated_at) { 30.minutes.ago } - - it_behaves_like 'job is unchanged' - end - end - - %w(success skipped failed canceled).each do |status| - context "when job is #{status}" do - let(:status) { status } - let(:updated_at) { 2.days.ago } - - context 'when created_at is the same as updated_at' do - let(:created_at) { 2.days.ago } - - it_behaves_like 'job is unchanged' - end - - context 'when created_at is before updated_at' do - let(:created_at) { 3.days.ago } - - it_behaves_like 'job is unchanged' - end - - context 'when created_at is outside lookback window' do - let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } - - it_behaves_like 'job is unchanged' - end - end - end - - context 'for deleted project' do - let(:status) { 'running' } - let(:updated_at) { 2.days.ago } - - before do - job.project.update!(pending_delete: true) - end - - it_behaves_like 'job is dropped' - end - - describe 'drop stale scheduled builds' do - let(:status) { 'scheduled' } - let(:updated_at) { } - - context 'when scheduled at 2 hours ago but it is not executed yet' do - let!(:job) { create(:ci_build, :scheduled, scheduled_at: 2.hours.ago) } - - it 'drops the stale scheduled build' do - expect(Ci::Build.scheduled.count).to eq(1) - expect(job).to be_scheduled - - service.execute - job.reload - - expect(Ci::Build.scheduled.count).to eq(0) - expect(job).to be_failed - expect(job).to be_stale_schedule - end - end - - context 'when scheduled at 30 minutes ago but it is not executed yet' do - let!(:job) { create(:ci_build, :scheduled, scheduled_at: 30.minutes.ago) } - - it 'does not drop the stale scheduled build yet' do - expect(Ci::Build.scheduled.count).to eq(1) - expect(job).to be_scheduled - - service.execute - - expect(Ci::Build.scheduled.count).to eq(1) - expect(job).to be_scheduled - end - end - - context 'when there are no stale scheduled builds' do - it 'does not drop the stale scheduled build yet' do - expect { service.execute }.not_to raise_error - end - end - end -end diff --git a/spec/services/ci/update_build_state_service_spec.rb b/spec/services/ci/update_build_state_service_spec.rb index 5bb3843da8f..e4dd3d0500f 100644 --- a/spec/services/ci/update_build_state_service_spec.rb +++ b/spec/services/ci/update_build_state_service_spec.rb @@ -112,6 +112,14 @@ RSpec.describe Ci::UpdateBuildStateService do .not_to have_received(:increment_trace_operation) .with(operation: :invalid) end + + it 'does not increment chunks_invalid_checksum trace metric' do + execute_with_stubbed_metrics! + + expect(metrics) + .not_to have_received(:increment_error_counter) + .with(type: :chunks_invalid_checksum) + end end context 'when build trace has been migrated' do @@ -174,6 +182,14 @@ RSpec.describe Ci::UpdateBuildStateService do .to have_received(:increment_trace_operation) .with(operation: :invalid) end + + it 'increments chunks_invalid_checksum trace metric' do + execute_with_stubbed_metrics! + + expect(metrics) + .to have_received(:increment_error_counter) + .with(type: :chunks_invalid_checksum) + end end context 'when trace checksum is valid' do @@ -191,6 +207,14 @@ RSpec.describe Ci::UpdateBuildStateService do expect(metrics) .not_to have_received(:increment_trace_operation) .with(operation: :corrupted) + + expect(metrics) + .not_to have_received(:increment_error_counter) + .with(type: :chunks_invalid_checksum) + + expect(metrics) + .not_to have_received(:increment_error_counter) + .with(type: :chunks_invalid_size) end context 'when using deprecated parameters' do @@ -208,6 +232,14 @@ RSpec.describe Ci::UpdateBuildStateService do expect(metrics) .not_to have_received(:increment_trace_operation) .with(operation: :corrupted) + + expect(metrics) + .not_to have_received(:increment_error_counter) + .with(type: :chunks_invalid_checksum) + + expect(metrics) + .not_to have_received(:increment_error_counter) + .with(type: :chunks_invalid_size) end end end @@ -227,6 +259,14 @@ RSpec.describe Ci::UpdateBuildStateService do expect(metrics) .to have_received(:increment_trace_operation) .with(operation: :corrupted) + + expect(metrics) + .to have_received(:increment_error_counter) + .with(type: :chunks_invalid_checksum) + + expect(metrics) + .to have_received(:increment_error_counter) + .with(type: :chunks_invalid_size) end end @@ -242,9 +282,17 @@ RSpec.describe Ci::UpdateBuildStateService do .to have_received(:increment_trace_operation) .with(operation: :invalid) + expect(metrics) + .to have_received(:increment_error_counter) + .with(type: :chunks_invalid_checksum) + expect(metrics) .not_to have_received(:increment_trace_operation) .with(operation: :corrupted) + + expect(metrics) + .not_to have_received(:increment_error_counter) + .with(type: :chunks_invalid_size) end end @@ -325,6 +373,10 @@ RSpec.describe Ci::UpdateBuildStateService do expect(metrics) .not_to have_received(:increment_trace_operation) .with(operation: :invalid) + + expect(metrics) + .not_to have_received(:increment_error_counter) + .with(type: :chunks_invalid_checksum) end context 'when build pending state is outdated' do diff --git a/spec/services/ci/update_pending_build_service_spec.rb b/spec/services/ci/update_pending_build_service_spec.rb index d842042de40..d36564938c8 100644 --- a/spec/services/ci/update_pending_build_service_spec.rb +++ b/spec/services/ci/update_pending_build_service_spec.rb @@ -3,21 +3,23 @@ require 'spec_helper' RSpec.describe Ci::UpdatePendingBuildService do - describe '#execute' do - let_it_be(:group) { create(:group) } - let_it_be(:project) { create(:project, namespace: group) } - let_it_be(:pending_build_1) { create(:ci_pending_build, project: project, instance_runners_enabled: false) } - let_it_be(:pending_build_2) { create(:ci_pending_build, project: project, instance_runners_enabled: true) } - let_it_be(:update_params) { { instance_runners_enabled: true } } + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, namespace: group) } + let_it_be_with_reload(:pending_build_1) { create(:ci_pending_build, project: project, instance_runners_enabled: false) } + let_it_be_with_reload(:pending_build_2) { create(:ci_pending_build, project: project, instance_runners_enabled: true) } + let_it_be(:update_params) { { instance_runners_enabled: true } } + + let(:service) { described_class.new(model, update_params) } - subject(:service) { described_class.new(model, update_params).execute } + describe '#execute' do + subject(:update_pending_builds) { service.execute } context 'validations' do context 'when model is invalid' do let(:model) { pending_build_1 } it 'raises an error' do - expect { service }.to raise_error(described_class::InvalidModelError) + expect { update_pending_builds }.to raise_error(described_class::InvalidModelError) end end @@ -26,7 +28,7 @@ RSpec.describe Ci::UpdatePendingBuildService do let(:update_params) { { minutes_exceeded: true } } it 'raises an error' do - expect { service }.to raise_error(described_class::InvalidParamsError) + expect { update_pending_builds }.to raise_error(described_class::InvalidParamsError) end end end @@ -35,10 +37,10 @@ RSpec.describe Ci::UpdatePendingBuildService do let(:model) { group } it 'updates all pending builds', :aggregate_failures do - service + update_pending_builds - expect(pending_build_1.reload.instance_runners_enabled).to be_truthy - expect(pending_build_2.reload.instance_runners_enabled).to be_truthy + expect(pending_build_1.instance_runners_enabled).to be_truthy + expect(pending_build_2.instance_runners_enabled).to be_truthy end context 'when ci_pending_builds_maintain_shared_runners_data is disabled' do @@ -47,10 +49,10 @@ RSpec.describe Ci::UpdatePendingBuildService do end it 'does not update all pending builds', :aggregate_failures do - service + update_pending_builds - expect(pending_build_1.reload.instance_runners_enabled).to be_falsey - expect(pending_build_2.reload.instance_runners_enabled).to be_truthy + expect(pending_build_1.instance_runners_enabled).to be_falsey + expect(pending_build_2.instance_runners_enabled).to be_truthy end end end @@ -59,10 +61,10 @@ RSpec.describe Ci::UpdatePendingBuildService do let(:model) { project } it 'updates all pending builds', :aggregate_failures do - service + update_pending_builds - expect(pending_build_1.reload.instance_runners_enabled).to be_truthy - expect(pending_build_2.reload.instance_runners_enabled).to be_truthy + expect(pending_build_1.instance_runners_enabled).to be_truthy + expect(pending_build_2.instance_runners_enabled).to be_truthy end context 'when ci_pending_builds_maintain_shared_runners_data is disabled' do @@ -71,10 +73,10 @@ RSpec.describe Ci::UpdatePendingBuildService do end it 'does not update all pending builds', :aggregate_failures do - service + update_pending_builds - expect(pending_build_1.reload.instance_runners_enabled).to be_falsey - expect(pending_build_2.reload.instance_runners_enabled).to be_truthy + expect(pending_build_1.instance_runners_enabled).to be_falsey + expect(pending_build_2.instance_runners_enabled).to be_truthy end end end -- cgit v1.2.1