Merge branch 'qa-team-tasks-92-test-failure-retries' into 'master'

Retry failing tests

Closes gitlab-org/quality/team-tasks#92

See merge request gitlab-org/gitlab-ce!25391

4 files changed, 67 insertions, 0 deletions

diff --git a/qa/Gemfile b/qa/Gemfile
index 873eac1013f..f29006617ed 100644
--- a/qa/Gemfile
+++ b/qa/Gemfile
@@ -8,3 +8,4 @@ gem 'rspec', '~> 3.7'
 gem 'selenium-webdriver', '~> 3.12'
 gem 'airborne', '~> 0.2.13'
 gem 'nokogiri', '~> 1.10.1'
+gem 'rspec-retry', '~> 0.6.1'
diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock
index 9f84bdc3828..c3d9f558c23 100644
--- a/qa/Gemfile.lock
+++ b/qa/Gemfile.lock
@@ -76,6 +76,8 @@ GEM
     rspec-mocks (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
+    rspec-retry (0.6.1)
+      rspec-core (> 3.3)
     rspec-support (3.7.0)
     rubyzip (1.2.2)
     selenium-webdriver (3.141.0)
@@ -101,6 +103,7 @@ DEPENDENCIES
   pry-byebug (~> 3.5.1)
   rake (~> 12.3.0)
   rspec (~> 3.7)
+  rspec-retry (~> 0.6.1)
   selenium-webdriver (~> 3.12)
 
 BUNDLED WITH
diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb
index 20a153f3f63..cbdd6e881b1 100644
--- a/qa/spec/spec_helper.rb
+++ b/qa/spec/spec_helper.rb
@@ -1,4 +1,5 @@
 require_relative '../qa'
+require 'rspec/retry'
 
 %w[helpers shared_examples].each do |d|
   Dir[::File.join(__dir__, d, '**', '*.rb')].each { |f| require f }
@@ -31,6 +32,17 @@ RSpec.configure do |config|
   config.profile_examples = 10
   config.order = :random
   Kernel.srand config.seed
+
+  # show retry status in spec process
+  config.verbose_retry = true
+
+  # show exception that triggers a retry if verbose_retry is set to true
+  config.display_try_failure_messages = true
+
+  config.around do |example|
+    retry_times = example.metadata.keys.include?(:quarantine) ? 1 : 3
+    example.run_with_retry retry: retry_times
+  end
 end
 
 # Skip tests in quarantine unless we explicitly focus on them.
diff --git a/qa/spec/spec_helper_spec.rb b/qa/spec/spec_helper_spec.rb
index 2427999e110..27ec1ec80fe 100644
--- a/qa/spec/spec_helper_spec.rb
+++ b/qa/spec/spec_helper_spec.rb
@@ -28,6 +28,22 @@ describe 'rspec config tests' do
     end
   end
 
+  let(:group_2) do
+    RSpec.describe do
+      before(:all) do
+        @expectations = [1, 2, 3]
+      end
+
+      example 'not in quarantine' do
+        expect(@expectations.shift).to be(3)
+      end
+
+      example 'in quarantine', :quarantine do
+        expect(@expectations.shift).to be(3)
+      end
+    end
+  end
+
   context 'with no tags focussed' do
     before do
       group.run
@@ -301,4 +317,39 @@ describe 'rspec config tests' do
       end
     end
   end
+
+  context 'rspec retry' do
+    context 'in an untagged context' do
+      before do
+        group_2.run
+      end
+
+      it 'should run example :retry times' do
+        examples = group_2.descendant_filtered_examples
+        ex = examples.find { |e| e.description == 'not in quarantine' }
+        expect(ex.execution_result.status).to eq(:passed)
+      end
+    end
+
+    context 'with :quarantine focussed' do
+      before do
+        RSpec.configure do |config|
+          config.inclusion_filter = :quarantine
+        end
+        group_2.run
+      end
+
+      after do
+        RSpec.configure do |config|
+          config.inclusion_filter.clear
+        end
+      end
+
+      it 'should run example once only' do
+        examples = group_2.descendant_filtered_examples
+        ex = examples.find { |e| e.description == 'in quarantine' }
+        expect(ex.execution_result.status).to eq(:failed)
+      end
+    end
+  end
 end