From c1918fb10b333593837c15bf4a6fa161ca502b4b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 17 Jul 2017 12:05:07 +0200 Subject: Add a new `retry` CI/CD configuration keyword --- lib/ci/gitlab_ci_yaml_processor.rb | 3 ++- lib/gitlab/ci/config/entry/job.rb | 10 ++++--- spec/lib/ci/gitlab_ci_yaml_processor_spec.rb | 22 ++++++++++++++++ spec/lib/gitlab/ci/config/entry/job_spec.rb | 39 ++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 4 deletions(-) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index cf3a0336792..3a4911b23b0 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -83,7 +83,8 @@ module Ci before_script: job[:before_script], script: job[:script], after_script: job[:after_script], - environment: job[:environment] + environment: job[:environment], + retry: job[:retry] }.compact } end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 176301bcca1..5f1144894b5 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,7 +11,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except type image services allow_failure type stage when artifacts cache dependencies before_script - after_script variables environment coverage].freeze + after_script variables environment coverage retry].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -23,6 +23,9 @@ module Gitlab with_options allow_nil: true do validates :tags, array_of_strings: true validates :allow_failure, boolean: true + validates :retry, numericality: { only_integer: true, + greater_than_or_equal_to: 0, + less_than: 10 } validates :when, inclusion: { in: %w[on_success on_failure always manual], message: 'should be on_success, on_failure, ' \ @@ -76,9 +79,9 @@ module Gitlab helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :commands, :environment, :coverage + :artifacts, :commands, :environment, :coverage, :retry - attributes :script, :tags, :allow_failure, :when, :dependencies + attributes :script, :tags, :allow_failure, :when, :dependencies, :retry def compose!(deps = nil) super do @@ -142,6 +145,7 @@ module Gitlab environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, + retry: retry_defined? ? retry_value.to_i : nil, artifacts: artifacts_value, after_script: after_script_value, ignore: ignored? } diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index ea79389e67e..e50f799a6e9 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -32,6 +32,28 @@ module Ci end end + describe 'retry entry' do + context 'when retry count is specified' do + let(:config) do + YAML.dump(rspec: { script: 'rspec', retry: 3 }) + end + + it 'includes retry count in build options attribute' do + expect(subject[:options]).to include(retry: 3) + end + end + + context 'when retry count is not specified' do + let(:config) do + YAML.dump(rspec: { script: 'rspec' }) + end + + it 'does not persist retry count in the database' do + expect(subject[:options]).not_to have_key(:retry) + end + end + end + describe 'allow failure entry' do context 'when job is a manual action' do context 'when allow_failure is defined' do diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index c5cad887b64..f8ed59a3a44 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -80,6 +80,45 @@ describe Gitlab::Ci::Config::Entry::Job do expect(entry.errors).to include "job script can't be blank" end end + + context 'when retry value is not correct' do + context 'when it is not a numeric value' do + let(:config) { { retry: true } } + + it 'returns error about invalid type' do + expect(entry).not_to be_valid + expect(entry.errors).to include 'job retry is not a number' + end + end + + context 'when it is lower than zero' do + let(:config) { { retry: -1 } } + + it 'returns error about value too low' do + expect(entry).not_to be_valid + expect(entry.errors) + .to include 'job retry must be greater than or equal to 0' + end + end + + context 'when it is not an integer' do + let(:config) { { retry: 1.5 } } + + it 'returns error about wrong value' do + expect(entry).not_to be_valid + expect(entry.errors).to include 'job retry must be an integer' + end + end + + context 'when the value is too high' do + let(:config) { { retry: 10 } } + + it 'returns error about value too high' do + expect(entry).not_to be_valid + expect(entry.errors).to include 'job retry must be less than 10' + end + end + end end end -- cgit v1.2.1 From 9bb7f19d15ac5412a1d4c816f4b3eebcb3c5a840 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 17 Jul 2017 12:38:21 +0200 Subject: Make it possible to count a number of job retries --- app/models/ci/build.rb | 8 ++++++++ spec/factories/ci/builds.rb | 4 ++++ spec/models/ci/build_spec.rb | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 432f3f242eb..24ba4a881f5 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -130,6 +130,14 @@ module Ci success? || failed? || canceled? end + def retries_count + pipeline.builds.retried.where(name: self.name).count + end + + def retries_max + self.options.fetch(:retry, 0).to_i + end + def latest? !retried? end diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb index a77f01ecb00..678cebe365b 100644 --- a/spec/factories/ci/builds.rb +++ b/spec/factories/ci/builds.rb @@ -84,6 +84,10 @@ FactoryGirl.define do success end + trait :retried do + retried true + end + trait :cancelable do pending end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 154b6759f46..615d1e09a11 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -802,6 +802,47 @@ describe Ci::Build, :models do end end + describe 'build auto retry feature' do + describe '#retries_count' do + subject { create(:ci_build, name: 'test', pipeline: pipeline) } + + context 'when build has been retried several times' do + before do + create(:ci_build, :retried, name: 'test', pipeline: pipeline) + create(:ci_build, :retried, name: 'test', pipeline: pipeline) + end + + it 'reports a correct retry count value' do + expect(subject.retries_count).to eq 2 + end + end + + context 'when build has not been retried' do + it 'returns zero' do + expect(subject.retries_count).to eq 0 + end + end + end + + describe '#retries_max' do + context 'when max retries value is defined' do + subject { create(:ci_build, options: { retry: 3 }) } + + it 'returns a number of configured max retries' do + expect(subject.retries_max).to eq 3 + end + end + + context 'when max retries value is not defined' do + subject { create(:ci_build) } + + it 'returns zero' do + expect(subject.retries_max).to eq 0 + end + end + end + end + describe '#keep_artifacts!' do let(:build) { create(:ci_build, artifacts_expire_at: Time.now + 7.days) } -- cgit v1.2.1 From 7fde7012c9126172097fae57969f1694f7cf5f05 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 17 Jul 2017 13:00:32 +0200 Subject: Make it possible to auto retry a failed CI/CD job --- app/models/ci/build.rb | 10 ++++++++++ spec/models/ci/build_spec.rb | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 24ba4a881f5..a1b1fbefa42 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -96,6 +96,16 @@ module Ci BuildSuccessWorker.perform_async(id) end end + + after_transition any => [:failed] do |build| + build.run_after_commit do + next if build.retries_max.zero? + + if build.retries_count < build.retries_max + Ci::Build.retry(build, build.user) + end + end + end end def detailed_status(current_user) diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 615d1e09a11..acfc888d944 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1624,7 +1624,7 @@ describe Ci::Build, :models do end end - describe 'State transition: any => [:pending]' do + describe 'state transition: any => [:pending]' do let(:build) { create(:ci_build, :created) } it 'queues BuildQueueWorker' do @@ -1633,4 +1633,35 @@ describe Ci::Build, :models do build.enqueue end end + + describe 'state transition when build fails' do + context 'when build is configured to be retried' do + subject { create(:ci_build, :running, options: { retry: 3 }) } + + it 'retries builds and assigns a same user to it' do + expect(described_class).to receive(:retry) + .with(subject, subject.user) + + subject.drop! + end + end + + context 'when build is not configured to be retried' do + subject { create(:ci_build, :running) } + + it 'does not retry build' do + expect(described_class).not_to receive(:retry) + + subject.drop! + end + + it 'does not count retries when not necessary' do + expect(described_class).not_to receive(:retry) + expect_any_instance_of(described_class) + .not_to receive(:retries_count) + + subject.drop! + end + end + end end -- cgit v1.2.1 From a4d301eed047afb800b810432680b8c9134fa40a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 17 Jul 2017 13:10:07 +0200 Subject: Add specs seeding jobs with auto-retries configured --- spec/services/ci/create_pipeline_service_spec.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index 77c07b71c68..69f52b06980 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -320,5 +320,19 @@ describe Ci::CreatePipelineService, :services do end.not_to change { Environment.count } end end + + context 'when builds with auto-retries are configured' do + before do + config = YAML.dump(rspec: { script: 'rspec', retry: 3 }) + stub_ci_pipeline_yaml_file(config) + end + + it 'correctly creates builds with auto-retry value configured' do + pipeline = execute_service + + expect(pipeline).to be_persisted + expect(pipeline.builds.find_by(name: 'rspec').retries_max).to eq 3 + end + end end end -- cgit v1.2.1 From 63137d4e5bd8e42d5b95ffac44be71f998148494 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 17 Jul 2017 13:30:49 +0200 Subject: Add specs for pipeline process with auto-retries This also resolve a possible race condition - a next stage should not start until are retries are done in a previous one. --- app/models/ci/build.rb | 10 ++++---- spec/services/ci/process_pipeline_service_spec.rb | 29 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index a1b1fbefa42..416a2a33378 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -97,13 +97,11 @@ module Ci end end - after_transition any => [:failed] do |build| - build.run_after_commit do - next if build.retries_max.zero? + before_transition any => [:failed] do |build| + next if build.retries_max.zero? - if build.retries_count < build.retries_max - Ci::Build.retry(build, build.user) - end + if build.retries_count < build.retries_max + Ci::Build.retry(build, build.user) end end end diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb index efcaccc254e..0934833a4fa 100644 --- a/spec/services/ci/process_pipeline_service_spec.rb +++ b/spec/services/ci/process_pipeline_service_spec.rb @@ -463,6 +463,35 @@ describe Ci::ProcessPipelineService, '#execute', :services do end end + context 'when builds with auto-retries are configured' do + before do + create_build('build:1', stage_idx: 0, user: user, options: { retry: 2 }) + create_build('test:1', stage_idx: 1, user: user, when: :on_failure) + create_build('test:2', stage_idx: 1, user: user, options: { retry: 1 }) + end + + it 'automatically retries builds in a valid order' do + expect(process_pipeline).to be_truthy + + fail_running_or_pending + + expect(builds_names).to eq %w[build:1 build:1] + expect(builds_statuses).to eq %w[failed pending] + + succeed_running_or_pending + + expect(builds_names).to eq %w[build:1 build:1 test:2] + expect(builds_statuses).to eq %w[failed success pending] + + succeed_running_or_pending + + expect(builds_names).to eq %w[build:1 build:1 test:2] + expect(builds_statuses).to eq %w[failed success success] + + expect(pipeline.reload).to be_success + end + end + def process_pipeline described_class.new(pipeline.project, user).execute(pipeline) end -- cgit v1.2.1 From 67157d81149856b90c5f0034e5e4bb05a35fbfcc Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 18 Jul 2017 12:54:24 +0200 Subject: Add changelog entry for auto-retry of CI/CD job feature --- changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml diff --git a/changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml b/changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml new file mode 100644 index 00000000000..abde2397331 --- /dev/null +++ b/changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml @@ -0,0 +1,4 @@ +--- +title: Make it possible to configure auto-retry of CI/CD job +merge_request: 12909 +author: -- cgit v1.2.1 From 883e8f8691af573246aced279b48cae46d8f2a1c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 18 Jul 2017 13:01:29 +0200 Subject: Add basic docs for CI/CD job auto-retry feature --- doc/ci/yaml/README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 724843a4d56..6b17af1394a 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -395,6 +395,7 @@ job_name: | after_script | no | Override a set of commands that are executed after job | | environment | no | Defines a name of environment to which deployment is done by this job | | coverage | no | Define code coverage settings for a given job | +| retry | no | Define how many times a job can be auto-retried in case of a failure | ### script @@ -1129,9 +1130,33 @@ A simple example: ```yaml job1: + script: rspec coverage: '/Code coverage: \d+\.\d+/' ``` +### retry + +**Notes:** +- [Introduced][ce-3442] in GitLab 9.5. + +`retry` allows you to configure how many times a job is going to be retried in +case of a failure. + +When a job fails, and has `retry` configured it is going to be processed again +up to the amount of times specified by the `retry` keyword. + +If `retry` is set to 3, and a job succeeds in a second run, it won't be retried +again. `retry` value has to be a positive integer, equal or larger than 0, but +lower than 10. + +A simple example: + +```yaml +test: + script: rspec + retry: 3 +``` + ## Git Strategy > Introduced in GitLab 8.9 as an experimental feature. May change or be removed @@ -1506,3 +1531,4 @@ CI with various languages. [variables]: ../variables/README.md [ce-7983]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7983 [ce-7447]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7447 +[ce-3442]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3442 -- cgit v1.2.1 From f49a906a5f805d44ce2b4c6c01e58a38a606761b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 19 Jul 2017 11:02:21 +0200 Subject: Update changelog entry for a auto-retry of CI/CD jobs --- changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml b/changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml index abde2397331..bdafc5929c0 100644 --- a/changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml +++ b/changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml @@ -1,4 +1,4 @@ --- -title: Make it possible to configure auto-retry of CI/CD job +title: Allow to configure automatic retry of a failed CI/CD job merge_request: 12909 author: -- cgit v1.2.1 From 23223ba64e91cc3406ed6cf5889f8cc25f9a1337 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 19 Jul 2017 11:27:49 +0200 Subject: Do not allow to auto-retry a job more than 2 times --- doc/ci/yaml/README.md | 2 +- lib/gitlab/ci/config/entry/job.rb | 2 +- spec/lib/gitlab/ci/config/entry/job_spec.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 6b17af1394a..808a23df554 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -1147,7 +1147,7 @@ up to the amount of times specified by the `retry` keyword. If `retry` is set to 3, and a job succeeds in a second run, it won't be retried again. `retry` value has to be a positive integer, equal or larger than 0, but -lower than 10. +lower or equal to 2 (two retries maximum, three runs in total). A simple example: diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 5f1144894b5..32f5c6ab142 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -25,7 +25,7 @@ module Gitlab validates :allow_failure, boolean: true validates :retry, numericality: { only_integer: true, greater_than_or_equal_to: 0, - less_than: 10 } + less_than_or_equal_to: 2 } validates :when, inclusion: { in: %w[on_success on_failure always manual], message: 'should be on_success, on_failure, ' \ diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index f8ed59a3a44..6769f64f950 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -115,7 +115,7 @@ describe Gitlab::Ci::Config::Entry::Job do it 'returns error about value too high' do expect(entry).not_to be_valid - expect(entry.errors).to include 'job retry must be less than 10' + expect(entry.errors).to include 'job retry must be less than or equal to 2' end end end -- cgit v1.2.1 From d382ed5eb618b5719f4fbf705a47b00c98e7c5a6 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 20 Jul 2017 07:43:15 +0200 Subject: Fix CI/CD job auto-retry specs --- spec/lib/ci/gitlab_ci_yaml_processor_spec.rb | 4 ++-- spec/services/ci/create_pipeline_service_spec.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index e50f799a6e9..ed571a2ba05 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -35,11 +35,11 @@ module Ci describe 'retry entry' do context 'when retry count is specified' do let(:config) do - YAML.dump(rspec: { script: 'rspec', retry: 3 }) + YAML.dump(rspec: { script: 'rspec', retry: 1 }) end it 'includes retry count in build options attribute' do - expect(subject[:options]).to include(retry: 3) + expect(subject[:options]).to include(retry: 1) end end diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index 69f52b06980..194332f62c6 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -323,7 +323,7 @@ describe Ci::CreatePipelineService, :services do context 'when builds with auto-retries are configured' do before do - config = YAML.dump(rspec: { script: 'rspec', retry: 3 }) + config = YAML.dump(rspec: { script: 'rspec', retry: 2 }) stub_ci_pipeline_yaml_file(config) end @@ -331,7 +331,7 @@ describe Ci::CreatePipelineService, :services do pipeline = execute_service expect(pipeline).to be_persisted - expect(pipeline.builds.find_by(name: 'rspec').retries_max).to eq 3 + expect(pipeline.builds.find_by(name: 'rspec').retries_max).to eq 2 end end end -- cgit v1.2.1 From 2f620aa7116f504229be81c2465fead342a57292 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 20 Jul 2017 11:02:00 +0200 Subject: Change auto-retry count to a correct value in docs --- doc/ci/yaml/README.md | 4 ++-- spec/models/ci/build_spec.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 808a23df554..e12ef6e2685 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -1145,7 +1145,7 @@ case of a failure. When a job fails, and has `retry` configured it is going to be processed again up to the amount of times specified by the `retry` keyword. -If `retry` is set to 3, and a job succeeds in a second run, it won't be retried +If `retry` is set to 2, and a job succeeds in a second run (first retry), it won't be retried again. `retry` value has to be a positive integer, equal or larger than 0, but lower or equal to 2 (two retries maximum, three runs in total). @@ -1154,7 +1154,7 @@ A simple example: ```yaml test: script: rspec - retry: 3 + retry: 2 ``` ## Git Strategy diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index acfc888d944..0b521d720f3 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -826,10 +826,10 @@ describe Ci::Build, :models do describe '#retries_max' do context 'when max retries value is defined' do - subject { create(:ci_build, options: { retry: 3 }) } + subject { create(:ci_build, options: { retry: 1 }) } it 'returns a number of configured max retries' do - expect(subject.retries_max).to eq 3 + expect(subject.retries_max).to eq 1 end end -- cgit v1.2.1