1 files changed, 52 insertions, 39 deletions

diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb
index 0f3cf5f4408..cbdd6e881b1 100644
--- a/qa/spec/spec_helper.rb
+++ b/qa/spec/spec_helper.rb
@@ -1,42 +1,21 @@
 require_relative '../qa'
+require 'rspec/retry'
 
-Dir[::File.join(__dir__, 'support', '**', '*.rb')].each { |f| require f }
+%w[helpers shared_examples].each do |d|
+  Dir[::File.join(__dir__, d, '**', '*.rb')].each { |f| require f }
+end
 
 RSpec.configure do |config|
-  ServerNotRespondingError = Class.new(RuntimeError)
-
-  # The login page could take some time to load the first time it is visited.
-  # We visit the login page and wait for it to properly load only once at the beginning of the suite.
-  config.before(:suite) do
-    if QA::Runtime::Scenario.respond_to?(:gitlab_address)
-      QA::Runtime::Browser.visit(:gitlab, QA::Page::Main::Login)
-
-      unless QA::Page::Main::Login.perform(&:page_loaded?)
-        raise ServerNotRespondingError, "Login page did not load at #{QA::Page::Main::Login.perform(&:current_url)}"
-      end
+  config.before(:context) do
+    if self.class.metadata.keys.include?(:quarantine)
+      skip_or_run_quarantined_tests(self.class.metadata.keys, config.inclusion_filter.rules.keys)
     end
   end
 
   config.before do |example|
     QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug?
 
-    # If quarantine is tagged, skip tests that have other metadata unless
-    # they're also tagged. This lets us run quarantined tests in a particular
-    # category without running tests in other categories.
-    # E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged
-    # 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests
-    # using `--tag quarantine --tag smoke`, without this check we'd end up
-    # running that ldap test as well.
-    if config.inclusion_filter[:quarantine]
-      skip("Running tests tagged with all of #{config.inclusion_filter.rules.keys}") unless quarantine_and_optional_other_tag?(example, config)
-    end
-  end
-
-  config.before(:each, :quarantine) do |example|
-    # Skip tests in quarantine unless we explicitly focus on them
-    # We could use an exclusion filter, but this way the test report will list
-    # the quarantined tests when they're not run so that we're aware of them
-    skip('In quarantine') unless config.inclusion_filter[:quarantine]
+    skip_or_run_quarantined_tests(example.metadata.keys, config.inclusion_filter.rules.keys)
   end
 
   config.expect_with :rspec do |expectations|
@@ -53,20 +32,54 @@ RSpec.configure do |config|
   config.profile_examples = 10
   config.order = :random
   Kernel.srand config.seed
+
+  # show retry status in spec process
+  config.verbose_retry = true
+
+  # show exception that triggers a retry if verbose_retry is set to true
+  config.display_try_failure_messages = true
+
+  config.around do |example|
+    retry_times = example.metadata.keys.include?(:quarantine) ? 1 : 3
+    example.run_with_retry retry: retry_times
+  end
+end
+
+# Skip tests in quarantine unless we explicitly focus on them.
+# Skip the entire context if a context is tagged. This avoids running before
+# blocks unnecessarily.
+# If quarantine is focussed, skip tests/contexts that have other metadata
+# unless they're also focussed. This lets us run quarantined tests in a
+# particular category without running tests in other categories.
+# E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged
+# 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests
+# using `--tag quarantine --tag smoke`, without this check we'd end up
+# running that ldap test as well.
+# We could use an exclusion filter, but this way the test report will list
+# the quarantined tests when they're not run so that we're aware of them
+def skip_or_run_quarantined_tests(metadata_keys, filter_keys)
+  included_filters = filters_other_than_quarantine(filter_keys)
+
+  if filter_keys.include?(:quarantine)
+    skip("Only running tests tagged with :quarantine and any of #{included_filters}") unless quarantine_and_optional_other_tag?(metadata_keys, included_filters)
+  else
+    skip('In quarantine') if metadata_keys.include?(:quarantine)
+  end
+end
+
+def filters_other_than_quarantine(filter_keys)
+  filter_keys.reject { |key| key == :quarantine }
 end
 
 # Checks if a test has the 'quarantine' tag and other tags in the inclusion filter.
 #
 # Returns true if
-# - the example metadata includes the quarantine tag
-# - and the metadata and inclusion filter both have any other tag
-# - or no other tags are in the inclusion filter
-def quarantine_and_optional_other_tag?(example, config)
-  return false unless example.metadata.keys.include? :quarantine
-
-  filters_other_than_quarantine = config.inclusion_filter.rules.keys.reject { |key| key == :quarantine }
-
-  return true if filters_other_than_quarantine.empty?
+# - the metadata includes the quarantine tag
+#   - and the metadata and inclusion filter both have any other tag
+#   - or no other tags are in the inclusion filter
+def quarantine_and_optional_other_tag?(metadata_keys, included_filters)
+  return false unless metadata_keys.include? :quarantine
+  return true if included_filters.empty?
 
-  filters_other_than_quarantine.any? { |key| example.metadata.keys.include? key }
+  included_filters.any? { |key| metadata_keys.include? key }
 end