Merge branch 'feature/gb/auto-retry-failed-ci-job' into 'master'

Make it possible to auto retry a failed CI/CD job

Closes #3442

See merge request !12909

11 files changed, 236 insertions, 5 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 432f3f242eb..416a2a33378 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -96,6 +96,14 @@ module Ci
           BuildSuccessWorker.perform_async(id)
         end
       end
+
+      before_transition any => [:failed] do |build|
+        next if build.retries_max.zero?
+
+        if build.retries_count < build.retries_max
+          Ci::Build.retry(build, build.user)
+        end
+      end
     end
 
     def detailed_status(current_user)
@@ -130,6 +138,14 @@ module Ci
       success? || failed? || canceled?
     end
 
+    def retries_count
+      pipeline.builds.retried.where(name: self.name).count
+    end
+
+    def retries_max
+      self.options.fetch(:retry, 0).to_i
+    end
+
     def latest?
       !retried?
     end
diff --git a/changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml b/changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml
new file mode 100644
index 00000000000..bdafc5929c0
--- /dev/null
+++ b/changelogs/unreleased/feature-gb-auto-retry-failed-ci-job.yml
@@ -0,0 +1,4 @@
+---
+title: Allow to configure automatic retry of a failed CI/CD job
+merge_request: 12909
+author:
diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md
index 724843a4d56..e12ef6e2685 100644
--- a/doc/ci/yaml/README.md
+++ b/doc/ci/yaml/README.md
@@ -395,6 +395,7 @@ job_name:
 | after_script  | no       | Override a set of commands that are executed after job |
 | environment   | no       | Defines a name of environment to which deployment is done by this job |
 | coverage      | no       | Define code coverage settings for a given job |
+| retry         | no       | Define how many times a job can be auto-retried in case of a failure |
 
 ### script
 
@@ -1129,9 +1130,33 @@ A simple example:
 
 ```yaml
 job1:
+  script: rspec
   coverage: '/Code coverage: \d+\.\d+/'
 ```
 
+### retry
+
+**Notes:**
+- [Introduced][ce-3442] in GitLab 9.5.
+
+`retry` allows you to configure how many times a job is going to be retried in
+case of a failure.
+
+When a job fails, and has `retry` configured it is going to be processed again
+up to the amount of times specified by the `retry` keyword.
+
+If `retry` is set to 2, and a job succeeds in a second run (first retry), it won't be retried
+again. `retry` value has to be a positive integer, equal or larger than 0, but
+lower or equal to 2 (two retries maximum, three runs in total).
+
+A simple example:
+
+```yaml
+test:
+  script: rspec
+  retry: 2
+```
+
 ## Git Strategy
 
 > Introduced in GitLab 8.9 as an experimental feature.  May change or be removed
@@ -1506,3 +1531,4 @@ CI with various languages.
 [variables]: ../variables/README.md
 [ce-7983]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7983
 [ce-7447]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7447
+[ce-3442]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3442
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index cf3a0336792..3a4911b23b0 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -83,7 +83,8 @@ module Ci
           before_script: job[:before_script],
           script: job[:script],
           after_script: job[:after_script],
-          environment: job[:environment]
+          environment: job[:environment],
+          retry: job[:retry]
         }.compact }
     end
 
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 176301bcca1..32f5c6ab142 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -11,7 +11,7 @@ module Gitlab
 
           ALLOWED_KEYS = %i[tags script only except type image services allow_failure
                             type stage when artifacts cache dependencies before_script
-                            after_script variables environment coverage].freeze
+                            after_script variables environment coverage retry].freeze
 
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS
@@ -23,6 +23,9 @@ module Gitlab
             with_options allow_nil: true do
               validates :tags, array_of_strings: true
               validates :allow_failure, boolean: true
+              validates :retry, numericality: { only_integer: true,
+                                                greater_than_or_equal_to: 0,
+                                                less_than_or_equal_to: 2 }
               validates :when,
                 inclusion: { in: %w[on_success on_failure always manual],
                              message: 'should be on_success, on_failure, ' \
@@ -76,9 +79,9 @@ module Gitlab
 
           helpers :before_script, :script, :stage, :type, :after_script,
                   :cache, :image, :services, :only, :except, :variables,
-                  :artifacts, :commands, :environment, :coverage
+                  :artifacts, :commands, :environment, :coverage, :retry
 
-          attributes :script, :tags, :allow_failure, :when, :dependencies
+          attributes :script, :tags, :allow_failure, :when, :dependencies, :retry
 
           def compose!(deps = nil)
             super do
@@ -142,6 +145,7 @@ module Gitlab
               environment: environment_defined? ? environment_value : nil,
               environment_name: environment_defined? ? environment_value[:name] : nil,
               coverage: coverage_defined? ? coverage_value : nil,
+              retry: retry_defined? ? retry_value.to_i : nil,
               artifacts: artifacts_value,
               after_script: after_script_value,
               ignore: ignored? }
diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb
index a77f01ecb00..678cebe365b 100644
--- a/spec/factories/ci/builds.rb
+++ b/spec/factories/ci/builds.rb
@@ -84,6 +84,10 @@ FactoryGirl.define do
       success
     end
 
+    trait :retried do
+      retried true
+    end
+
     trait :cancelable do
       pending
     end
diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb
index ea79389e67e..ed571a2ba05 100644
--- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb
+++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb
@@ -32,6 +32,28 @@ module Ci
         end
       end
 
+      describe 'retry entry' do
+        context 'when retry count is specified' do
+          let(:config) do
+            YAML.dump(rspec: { script: 'rspec', retry: 1 })
+          end
+
+          it 'includes retry count in build options attribute' do
+            expect(subject[:options]).to include(retry: 1)
+          end
+        end
+
+        context 'when retry count is not specified' do
+          let(:config) do
+            YAML.dump(rspec: { script: 'rspec' })
+          end
+
+          it 'does not persist retry count in the database' do
+            expect(subject[:options]).not_to have_key(:retry)
+          end
+        end
+      end
+
       describe 'allow failure entry' do
         context 'when job is a manual action' do
           context 'when allow_failure is defined' do
diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb
index c5cad887b64..6769f64f950 100644
--- a/spec/lib/gitlab/ci/config/entry/job_spec.rb
+++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb
@@ -80,6 +80,45 @@ describe Gitlab::Ci::Config::Entry::Job do
           expect(entry.errors).to include "job script can't be blank"
         end
       end
+
+      context 'when retry value is not correct' do
+        context 'when it is not a numeric value' do
+          let(:config) { { retry: true } }
+
+          it 'returns error about invalid type' do
+            expect(entry).not_to be_valid
+            expect(entry.errors).to include 'job retry is not a number'
+          end
+        end
+
+        context 'when it is lower than zero' do
+          let(:config) { { retry: -1 } }
+
+          it 'returns error about value too low' do
+            expect(entry).not_to be_valid
+            expect(entry.errors)
+              .to include 'job retry must be greater than or equal to 0'
+          end
+        end
+
+        context 'when it is not an integer' do
+          let(:config) { { retry: 1.5 } }
+
+          it 'returns error about wrong value' do
+            expect(entry).not_to be_valid
+            expect(entry.errors).to include 'job retry must be an integer'
+          end
+        end
+
+        context 'when the value is too high' do
+          let(:config) { { retry: 10 } }
+
+          it 'returns error about value too high' do
+            expect(entry).not_to be_valid
+            expect(entry.errors).to include 'job retry must be less than or equal to 2'
+          end
+        end
+      end
     end
   end
 
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index 154b6759f46..0b521d720f3 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -802,6 +802,47 @@ describe Ci::Build, :models do
     end
   end
 
+  describe 'build auto retry feature' do
+    describe '#retries_count' do
+      subject { create(:ci_build, name: 'test', pipeline: pipeline) }
+
+      context 'when build has been retried several times' do
+        before do
+          create(:ci_build, :retried, name: 'test', pipeline: pipeline)
+          create(:ci_build, :retried, name: 'test', pipeline: pipeline)
+        end
+
+        it 'reports a correct retry count value' do
+          expect(subject.retries_count).to eq 2
+        end
+      end
+
+      context 'when build has not been retried' do
+        it 'returns zero' do
+          expect(subject.retries_count).to eq 0
+        end
+      end
+    end
+
+    describe '#retries_max' do
+      context 'when max retries value is defined' do
+        subject { create(:ci_build, options: { retry: 1 }) }
+
+        it 'returns a number of configured max retries' do
+          expect(subject.retries_max).to eq 1
+        end
+      end
+
+      context 'when max retries value is not defined' do
+        subject { create(:ci_build) }
+
+        it 'returns zero' do
+          expect(subject.retries_max).to eq 0
+        end
+      end
+    end
+  end
+
   describe '#keep_artifacts!' do
     let(:build) { create(:ci_build, artifacts_expire_at: Time.now + 7.days) }
 
@@ -1583,7 +1624,7 @@ describe Ci::Build, :models do
     end
   end
 
-  describe 'State transition: any => [:pending]' do
+  describe 'state transition: any => [:pending]' do
     let(:build) { create(:ci_build, :created) }
 
     it 'queues BuildQueueWorker' do
@@ -1592,4 +1633,35 @@ describe Ci::Build, :models do
       build.enqueue
     end
   end
+
+  describe 'state transition when build fails' do
+    context 'when build is configured to be retried' do
+      subject { create(:ci_build, :running, options: { retry: 3 }) }
+
+      it 'retries builds and assigns a same user to it' do
+        expect(described_class).to receive(:retry)
+          .with(subject, subject.user)
+
+        subject.drop!
+      end
+    end
+
+    context 'when build is not configured to be retried' do
+      subject { create(:ci_build, :running) }
+
+      it 'does not retry build' do
+        expect(described_class).not_to receive(:retry)
+
+        subject.drop!
+      end
+
+      it 'does not count retries when not necessary' do
+        expect(described_class).not_to receive(:retry)
+        expect_any_instance_of(described_class)
+          .not_to receive(:retries_count)
+
+        subject.drop!
+      end
+    end
+  end
 end
diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb
index e71c462b99a..ba07c01d43f 100644
--- a/spec/services/ci/create_pipeline_service_spec.rb
+++ b/spec/services/ci/create_pipeline_service_spec.rb
@@ -320,5 +320,19 @@ describe Ci::CreatePipelineService, :services do
         end.not_to change { Environment.count }
       end
     end
+
+    context 'when builds with auto-retries are configured' do
+      before do
+        config = YAML.dump(rspec: { script: 'rspec', retry: 2 })
+        stub_ci_pipeline_yaml_file(config)
+      end
+
+      it 'correctly creates builds with auto-retry value configured' do
+        pipeline = execute_service
+
+        expect(pipeline).to be_persisted
+        expect(pipeline.builds.find_by(name: 'rspec').retries_max).to eq 2
+      end
+    end
   end
 end
diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb
index efcaccc254e..0934833a4fa 100644
--- a/spec/services/ci/process_pipeline_service_spec.rb
+++ b/spec/services/ci/process_pipeline_service_spec.rb
@@ -463,6 +463,35 @@ describe Ci::ProcessPipelineService, '#execute', :services do
     end
   end
 
+  context 'when builds with auto-retries are configured' do
+    before do
+      create_build('build:1', stage_idx: 0, user: user, options: { retry: 2 })
+      create_build('test:1', stage_idx: 1, user: user, when: :on_failure)
+      create_build('test:2', stage_idx: 1, user: user, options: { retry: 1 })
+    end
+
+    it 'automatically retries builds in a valid order' do
+      expect(process_pipeline).to be_truthy
+
+      fail_running_or_pending
+
+      expect(builds_names).to eq %w[build:1 build:1]
+      expect(builds_statuses).to eq %w[failed pending]
+
+      succeed_running_or_pending
+
+      expect(builds_names).to eq %w[build:1 build:1 test:2]
+      expect(builds_statuses).to eq %w[failed success pending]
+
+      succeed_running_or_pending
+
+      expect(builds_names).to eq %w[build:1 build:1 test:2]
+      expect(builds_statuses).to eq %w[failed success success]
+
+      expect(pipeline.reload).to be_success
+    end
+  end
+
   def process_pipeline
     described_class.new(pipeline.project, user).execute(pipeline)
   end