Skip contexts in quarantine

This avoids running before/after blocks for tests that are in quarantine

3 files changed, 272 insertions, 401 deletions

diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb
index cbdd6e881b1..be13c3fb683 100644
--- a/qa/spec/spec_helper.rb
+++ b/qa/spec/spec_helper.rb
@@ -6,16 +6,10 @@ require 'rspec/retry'
 end
 
 RSpec.configure do |config|
-  config.before(:context) do
-    if self.class.metadata.keys.include?(:quarantine)
-      skip_or_run_quarantined_tests(self.class.metadata.keys, config.inclusion_filter.rules.keys)
-    end
-  end
+  QA::Specs::Helpers::Quarantine.configure_rspec
 
   config.before do |example|
     QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug?
-
-    skip_or_run_quarantined_tests(example.metadata.keys, config.inclusion_filter.rules.keys)
   end
 
   config.expect_with :rspec do |expectations|
@@ -44,42 +38,3 @@ RSpec.configure do |config|
     example.run_with_retry retry: retry_times
   end
 end
-
-# Skip tests in quarantine unless we explicitly focus on them.
-# Skip the entire context if a context is tagged. This avoids running before
-# blocks unnecessarily.
-# If quarantine is focussed, skip tests/contexts that have other metadata
-# unless they're also focussed. This lets us run quarantined tests in a
-# particular category without running tests in other categories.
-# E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged
-# 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests
-# using `--tag quarantine --tag smoke`, without this check we'd end up
-# running that ldap test as well.
-# We could use an exclusion filter, but this way the test report will list
-# the quarantined tests when they're not run so that we're aware of them
-def skip_or_run_quarantined_tests(metadata_keys, filter_keys)
-  included_filters = filters_other_than_quarantine(filter_keys)
-
-  if filter_keys.include?(:quarantine)
-    skip("Only running tests tagged with :quarantine and any of #{included_filters}") unless quarantine_and_optional_other_tag?(metadata_keys, included_filters)
-  else
-    skip('In quarantine') if metadata_keys.include?(:quarantine)
-  end
-end
-
-def filters_other_than_quarantine(filter_keys)
-  filter_keys.reject { |key| key == :quarantine }
-end
-
-# Checks if a test has the 'quarantine' tag and other tags in the inclusion filter.
-#
-# Returns true if
-# - the metadata includes the quarantine tag
-#   - and the metadata and inclusion filter both have any other tag
-#   - or no other tags are in the inclusion filter
-def quarantine_and_optional_other_tag?(metadata_keys, included_filters)
-  return false unless metadata_keys.include? :quarantine
-  return true if included_filters.empty?
-
-  included_filters.any? { |key| metadata_keys.include? key }
-end
diff --git a/qa/spec/spec_helper_spec.rb b/qa/spec/spec_helper_spec.rb
deleted file mode 100644
index 27ec1ec80fe..00000000000
--- a/qa/spec/spec_helper_spec.rb
+++ /dev/null
@@ -1,355 +0,0 @@
-# frozen_string_literal: true
-
-describe 'rspec config tests' do
-  let(:group) do
-    RSpec.describe do
-      shared_examples 'passing tests' do
-        example 'not in quarantine' do
-        end
-        example 'in quarantine', :quarantine do
-        end
-      end
-
-      context 'default' do
-        it_behaves_like 'passing tests'
-      end
-
-      context 'foo', :foo do
-        it_behaves_like 'passing tests'
-      end
-
-      context 'quarantine', :quarantine do
-        it_behaves_like 'passing tests'
-      end
-
-      context 'bar quarantine', :bar, :quarantine do
-        it_behaves_like 'passing tests'
-      end
-    end
-  end
-
-  let(:group_2) do
-    RSpec.describe do
-      before(:all) do
-        @expectations = [1, 2, 3]
-      end
-
-      example 'not in quarantine' do
-        expect(@expectations.shift).to be(3)
-      end
-
-      example 'in quarantine', :quarantine do
-        expect(@expectations.shift).to be(3)
-      end
-    end
-  end
-
-  context 'with no tags focussed' do
-    before do
-      group.run
-    end
-
-    context 'in a context tagged :foo' do
-      it 'skips tests in quarantine' do
-        context = group.children.find { |c| c.description == "foo" }
-        examples = context.descendant_filtered_examples
-        expect(examples.count).to eq(2)
-
-        ex = examples.find { |e| e.description == "not in quarantine" }
-        expect(ex.execution_result.status).to eq(:passed)
-
-        ex = examples.find { |e| e.description == "in quarantine" }
-        expect(ex.execution_result.status).to eq(:pending)
-        expect(ex.execution_result.pending_message).to eq('In quarantine')
-      end
-    end
-
-    context 'in an untagged context' do
-      it 'skips tests in quarantine' do
-        context = group.children.find { |c| c.description == "default" }
-        examples = context.descendant_filtered_examples
-        expect(examples.count).to eq(2)
-
-        ex = examples.find { |e| e.description == "not in quarantine" }
-        expect(ex.execution_result.status).to eq(:passed)
-
-        ex = examples.find { |e| e.description == "in quarantine" }
-        expect(ex.execution_result.status).to eq(:pending)
-        expect(ex.execution_result.pending_message).to eq('In quarantine')
-      end
-    end
-
-    context 'in a context tagged :quarantine' do
-      it 'skips all tests' do
-        context = group.children.find { |c| c.description == "quarantine" }
-        examples = context.descendant_filtered_examples
-        expect(examples.count).to eq(2)
-
-        ex = examples.find { |e| e.description == "not in quarantine" }
-        expect(ex.execution_result.status).to eq(:pending)
-
-        ex = examples.find { |e| e.description == "in quarantine" }
-        expect(ex.execution_result.status).to eq(:pending)
-        expect(ex.execution_result.pending_message).to eq('In quarantine')
-      end
-    end
-  end
-
-  context 'with :quarantine focussed' do
-    before do
-      RSpec.configure do |config|
-        config.inclusion_filter = :quarantine
-      end
-
-      group.run
-    end
-    after do
-      RSpec.configure do |config|
-        config.inclusion_filter.clear
-      end
-    end
-
-    context 'in an untagged context' do
-      it 'only runs quarantined tests' do
-        context = group.children.find { |c| c.description == "default" }
-        examples = context.descendant_filtered_examples
-        expect(examples.count).to be(1)
-
-        ex = examples.find { |e| e.description == "in quarantine" }
-        expect(ex.execution_result.status).to eq(:passed)
-      end
-    end
-
-    context 'in a context tagged :foo' do
-      it 'only runs quarantined tests' do
-        context = group.children.find { |c| c.description == "foo" }
-        examples = context.descendant_filtered_examples
-        expect(examples.count).to be(1)
-
-        ex = examples.find { |e| e.description == "in quarantine" }
-        expect(ex.execution_result.status).to eq(:passed)
-      end
-    end
-
-    context 'in a context tagged :quarantine' do
-      it 'runs all tests' do
-        context = group.children.find { |c| c.description == "quarantine" }
-        examples = context.descendant_filtered_examples
-        expect(examples.count).to be(2)
-
-        ex = examples.find { |e| e.description == "in quarantine" }
-        expect(ex.execution_result.status).to eq(:passed)
-
-        ex = examples.find { |e| e.description == "not in quarantine" }
-        expect(ex.execution_result.status).to eq(:passed)
-      end
-    end
-  end
-
-  context 'with a non-quarantine tag (:foo) focussed' do
-    before do
-      RSpec.configure do |config|
-        config.inclusion_filter = :foo
-      end
-
-      group.run
-    end
-    after do
-      RSpec.configure do |config|
-        config.inclusion_filter.clear
-      end
-    end
-
-    context 'in an untagged context' do
-      it 'runs no tests' do
-        context = group.children.find { |c| c.description == "default" }
-        expect(context.descendant_filtered_examples.count).to eq(0)
-      end
-    end
-
-    context 'in a context tagged :foo' do
-      it 'skips quarantined tests' do
-        context = group.children.find { |c| c.description == "foo" }
-        examples = context.descendant_filtered_examples
-        expect(examples.count).to be(2)
-
-        ex = examples.find { |e| e.description == "not in quarantine" }
-        expect(ex.execution_result.status).to eq(:passed)
-
-        ex = examples.find { |e| e.description == "in quarantine" }
-        expect(ex.execution_result.status).to eq(:pending)
-        expect(ex.execution_result.pending_message).to eq('In quarantine')
-      end
-    end
-
-    context 'in a context tagged :quarantine' do
-      it 'runs no tests' do
-        context = group.children.find { |c| c.description == "quarantine" }
-        expect(context.descendant_filtered_examples.count).to eq(0)
-      end
-    end
-  end
-
-  context 'with :quarantine and a non-quarantine tag (:foo) focussed' do
-    before do
-      RSpec.configure do |config|
-        config.inclusion_filter = { quarantine: true, foo: true }
-      end
-
-      group.run
-    end
-    after do
-      RSpec.configure do |config|
-        config.inclusion_filter.clear
-      end
-    end
-
-    context 'in an untagged context' do
-      it 'ignores untagged tests and skips tests even if in quarantine' do
-        context = group.children.find { |c| c.description == "default" }
-        examples = context.descendant_filtered_examples
-        expect(examples.count).to eq(1)
-
-        ex = examples.find { |e| e.description == "in quarantine" }
-        expect(ex.execution_result.status).to eq(:pending)
-      end
-    end
-
-    context 'in a context tagged :foo' do
-      it 'only runs quarantined tests' do
-        context = group.children.find { |c| c.description == "foo" }
-        examples = context.descendant_filtered_examples
-        expect(examples.count).to be(2)
-
-        ex = examples.find { |e| e.description == "in quarantine" }
-        expect(ex.execution_result.status).to eq(:passed)
-
-        ex = examples.find { |e| e.description == "not in quarantine" }
-        expect(ex.execution_result.status).to eq(:pending)
-      end
-    end
-
-    context 'in a context tagged :quarantine' do
-      it 'skips all tests' do
-        context = group.children.find { |c| c.description == "quarantine" }
-        examples = context.descendant_filtered_examples
-        expect(examples.count).to be(2)
-
-        ex = examples.find { |e| e.description == "in quarantine" }
-        expect(ex.execution_result.status).to eq(:pending)
-
-        ex = examples.find { |e| e.description == "not in quarantine" }
-        expect(ex.execution_result.status).to eq(:pending)
-      end
-    end
-
-    context 'in a context tagged :bar and :quarantine' do
-      it 'skips all tests' do
-        context = group.children.find { |c| c.description == "quarantine" }
-        examples = context.descendant_filtered_examples
-        expect(examples.count).to be(2)
-
-        ex = examples.find { |e| e.description == "in quarantine" }
-        expect(ex.execution_result.status).to eq(:pending)
-
-        ex = examples.find { |e| e.description == "not in quarantine" }
-        expect(ex.execution_result.status).to eq(:pending)
-      end
-    end
-  end
-
-  context 'with :quarantine and multiple non-quarantine tags focussed' do
-    before do
-      RSpec.configure do |config|
-        config.inclusion_filter = { bar: true, foo: true, quarantine: true }
-      end
-
-      group.run
-    end
-    after do
-      RSpec.configure do |config|
-        config.inclusion_filter.clear
-      end
-    end
-
-    context 'in a context tagged :foo' do
-      it 'only runs quarantined tests' do
-        context = group.children.find { |c| c.description == "foo" }
-        examples = context.descendant_filtered_examples
-        expect(examples.count).to be(2)
-
-        ex = examples.find { |e| e.description == "in quarantine" }
-        expect(ex.execution_result.status).to eq(:passed)
-
-        ex = examples.find { |e| e.description == "not in quarantine" }
-        expect(ex.execution_result.status).to eq(:pending)
-        expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
-      end
-    end
-
-    context 'in a context tagged :quarantine' do
-      it 'skips all tests' do
-        context = group.children.find { |c| c.description == "quarantine" }
-        examples = context.descendant_filtered_examples
-        expect(examples.count).to be(2)
-
-        ex = examples.find { |e| e.description == "in quarantine" }
-        expect(ex.execution_result.status).to eq(:pending)
-        expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
-
-        ex = examples.find { |e| e.description == "not in quarantine" }
-        expect(ex.execution_result.status).to eq(:pending)
-        expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
-      end
-    end
-
-    context 'in a context tagged :bar and :quarantine' do
-      it 'runs all tests' do
-        context = group.children.find { |c| c.description == "bar quarantine" }
-        examples = context.descendant_filtered_examples
-        expect(examples.count).to be(2)
-
-        ex = examples.find { |e| e.description == "in quarantine" }
-        expect(ex.execution_result.status).to eq(:passed)
-
-        ex = examples.find { |e| e.description == "not in quarantine" }
-        expect(ex.execution_result.status).to eq(:passed)
-      end
-    end
-  end
-
-  context 'rspec retry' do
-    context 'in an untagged context' do
-      before do
-        group_2.run
-      end
-
-      it 'should run example :retry times' do
-        examples = group_2.descendant_filtered_examples
-        ex = examples.find { |e| e.description == 'not in quarantine' }
-        expect(ex.execution_result.status).to eq(:passed)
-      end
-    end
-
-    context 'with :quarantine focussed' do
-      before do
-        RSpec.configure do |config|
-          config.inclusion_filter = :quarantine
-        end
-        group_2.run
-      end
-
-      after do
-        RSpec.configure do |config|
-          config.inclusion_filter.clear
-        end
-      end
-
-      it 'should run example once only' do
-        examples = group_2.descendant_filtered_examples
-        ex = examples.find { |e| e.description == 'in quarantine' }
-        expect(ex.execution_result.status).to eq(:failed)
-      end
-    end
-  end
-end
diff --git a/qa/spec/specs/helpers/quarantine_spec.rb b/qa/spec/specs/helpers/quarantine_spec.rb
new file mode 100644
index 00000000000..78beda39b5e
--- /dev/null
+++ b/qa/spec/specs/helpers/quarantine_spec.rb
@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+
+require 'rspec/core/sandbox'
+
+# We need a reporter for internal tests that's different from the reporter for
+# external tests otherwise the results will be mixed up. We don't care about
+# most reporting, but we do want to know if a test fails
+class RaiseOnFailuresReporter < RSpec::Core::NullReporter
+  def self.example_failed(example)
+    raise example.exception
+  end
+end
+
+# We use an example group wrapper to prevent the state of internal tests
+# expanding into the global state
+# See: https://github.com/rspec/rspec-core/issues/2603
+def describe_successfully(*args, &describe_body)
+  example_group    = RSpec.describe(*args, &describe_body)
+  ran_successfully = example_group.run RaiseOnFailuresReporter
+  expect(ran_successfully).to eq true
+  example_group
+end
+
+RSpec.configure do |c|
+  c.around do |ex|
+    RSpec::Core::Sandbox.sandboxed do |config|
+      # If there is an example-within-an-example, we want to make sure the inner example
+      # does not get a reference to the outer example (the real spec) if it calls
+      # something like `pending`
+      config.before(:context) { RSpec.current_example = nil }
+
+      config.color_mode = :off
+
+      # Load airborne again to avoid "undefined method `match_expected_default?'" errors
+      # that happen because a hook calls a method added via a custom RSpec setting
+      # that is removed when the RSpec configuration is sandboxed.
+      # If this needs to be changed (e.g., to load other libraries as well), see
+      # this discussion for alternative solutions:
+      # https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/25223#note_143392053
+      load 'airborne.rb'
+
+      ex.run
+    end
+  end
+end
+
+describe QA::Specs::Helpers::Quarantine do
+  describe '.skip_or_run_quarantined_contexts' do
+    context 'with no tag focused' do
+      before do
+        described_class.configure_rspec
+      end
+
+      it 'skips before hooks of quarantined contexts' do
+        executed_hooks = []
+
+        group = describe_successfully('quarantine', :quarantine) do
+          before(:all) do
+            executed_hooks << :before_all
+          end
+          before do
+            executed_hooks << :before
+          end
+          example {}
+        end
+
+        expect(executed_hooks).to eq []
+        expect(group.descendant_filtered_examples.first.execution_result.status).to eq(:pending)
+        expect(group.descendant_filtered_examples.first.execution_result.pending_message)
+          .to eq('In quarantine')
+      end
+
+      it 'executes before hooks of non-quarantined contexts' do
+        executed_hooks = []
+
+        group = describe_successfully do
+          before(:all) do
+            executed_hooks << :before_all
+          end
+          before do
+            executed_hooks << :before
+          end
+          example {}
+        end
+
+        expect(executed_hooks).to eq [:before_all, :before]
+        expect(group.descendant_filtered_examples.first.execution_result.status).to eq(:passed)
+      end
+    end
+
+    context 'with :quarantine focused' do
+      before do
+        described_class.configure_rspec
+        RSpec.configure do |c|
+          c.filter_run :quarantine
+        end
+      end
+
+      it 'executes before hooks of quarantined contexts' do
+        executed_hooks = []
+
+        group = describe_successfully('quarantine', :quarantine) do
+          before(:all) do
+            executed_hooks << :before_all
+          end
+          before do
+            executed_hooks << :before
+          end
+          example {}
+        end
+
+        expect(executed_hooks).to eq [:before_all, :before]
+        expect(group.descendant_filtered_examples.first.execution_result.status).to eq(:passed)
+      end
+
+      it 'skips before hooks of non-quarantined contexts' do
+        executed_hooks = []
+
+        group = describe_successfully do
+          before(:all) do
+            executed_hooks << :before_all
+          end
+          before do
+            executed_hooks << :before
+          end
+          example {}
+        end
+
+        expect(executed_hooks).to eq []
+        expect(group.descendant_filtered_examples.first).to be_nil
+      end
+    end
+  end
+
+  describe '.skip_or_run_quarantined_tests' do
+    context 'with no tag focused' do
+      before do
+        described_class.configure_rspec
+      end
+
+      it 'skips quarantined tests' do
+        group = describe_successfully do
+          it('is pending', :quarantine) {}
+        end
+
+        expect(group.examples.first.execution_result.status).to eq(:pending)
+        expect(group.examples.first.execution_result.pending_message)
+          .to eq('In quarantine')
+      end
+
+      it 'executes non-quarantined tests' do
+        group = describe_successfully do
+          example {}
+        end
+
+        expect(group.examples.first.execution_result.status).to eq(:passed)
+      end
+    end
+
+    context 'with :quarantine focused' do
+      before do
+        described_class.configure_rspec
+        RSpec.configure do |c|
+          c.filter_run :quarantine
+        end
+      end
+
+      it 'executes quarantined tests' do
+        group = describe_successfully do
+          it('passes', :quarantine) {}
+        end
+
+        expect(group.examples.first.execution_result.status).to eq(:passed)
+      end
+
+      it 'ignores non-quarantined tests' do
+        group = describe_successfully do
+          example {}
+        end
+
+        expect(group.examples.first.execution_result.status).to be_nil
+      end
+    end
+
+    context 'with a non-quarantine tag focused' do
+      before do
+        described_class.configure_rspec
+        RSpec.configure do |c|
+          c.filter_run :foo
+        end
+      end
+
+      it 'ignores non-quarantined non-focused tests' do
+        group = describe_successfully do
+          example {}
+        end
+
+        expect(group.examples.first.execution_result.status).to be_nil
+      end
+
+      it 'executes non-quarantined focused tests' do
+        group = describe_successfully do
+          it('passes', :foo) {}
+        end
+
+        expect(group.examples.first.execution_result.status).to be(:passed)
+      end
+
+      it 'ignores quarantined tests' do
+        group = describe_successfully do
+          it('is ignored', :quarantine) {}
+        end
+
+        expect(group.examples.first.execution_result.status).to be_nil
+      end
+
+      it 'skips quarantined focused tests' do
+        group = describe_successfully do
+          it('is pending', :quarantine, :foo) {}
+        end
+
+        expect(group.examples.first.execution_result.status).to be(:pending)
+        expect(group.examples.first.execution_result.pending_message)
+          .to eq('In quarantine')
+      end
+    end
+
+    context 'with :quarantine and non-quarantine tags focused' do
+      before do
+        described_class.configure_rspec
+        RSpec.configure do |c|
+          c.filter_run :foo, :bar, :quarantine
+        end
+      end
+
+      it 'ignores non-quarantined non-focused tests' do
+        group = describe_successfully do
+          example {}
+        end
+
+        expect(group.examples.first.execution_result.status).to be_nil
+      end
+
+      it 'skips non-quarantined focused tests' do
+        group = describe_successfully do
+          it('is pending', :foo) {}
+        end
+
+        expect(group.examples.first.execution_result.status).to be(:pending)
+        expect(group.examples.first.execution_result.pending_message)
+          .to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
+      end
+
+      it 'skips quarantined non-focused tests' do
+        group = describe_successfully do
+          it('is pending', :quarantine) {}
+        end
+
+        expect(group.examples.first.execution_result.status).to be(:pending)
+      end
+
+      it 'executes quarantined focused tests' do
+        group = describe_successfully do
+          it('passes', :quarantine, :foo) {}
+        end
+
+        expect(group.examples.first.execution_result.status).to be(:passed)
+      end
+    end
+  end
+end