Refactor rspec config quarantine before blocksqa-ml-quarantine-context

And add tests for quarantined contexts

2 files changed, 220 insertions, 177 deletions

diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb
index c5c33596d55..f1b4203d422 100644
--- a/qa/spec/spec_helper.rb
+++ b/qa/spec/spec_helper.rb
@@ -19,35 +19,16 @@ RSpec.configure do |config|
     end
   end
 
-  # If quarantine is focussed, skip tests/contexts that have other metadata
-  # unless they're also focussed. This lets us run quarantined tests in a
-  # particular category without running tests in other categories.
-  # E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged
-  # 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests
-  # using `--tag quarantine --tag smoke`, without this check we'd end up
-  # running that ldap test as well.
-  config.before(:context, :quarantine) do
-    if config.inclusion_filter[:quarantine] && self.class.metadata.keys.include?(:quarantine)
-      skip("Only running contexts tagged with :quarantine and any of #{filters_other_than_quarantine(config)}") unless quarantine_and_optional_other_tag?(self.class.metadata.keys, config)
+  config.before(:context) do
+    if self.class.metadata.keys.include?(:quarantine)
+      skip_or_run_quarantined_tests(self.class.metadata.keys, config.inclusion_filter.rules.keys)
     end
   end
+
   config.before do |example|
     QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug?
 
-    if config.inclusion_filter[:quarantine]
-      skip("Only running tests tagged with :quarantine and any of #{filters_other_than_quarantine(config)}") unless quarantine_and_optional_other_tag?(example.metadata.keys, config)
-    end
-  end
-
-  # Skip tests in quarantine unless we explicitly focus on them.
-  # Skip the entire context if a context is tagged. This avoids running before
-  # blocks unnecessarily.
-  # We could use an exclusion filter, but this way the test report will list
-  # the quarantined tests when they're not run so that we're aware of them
-  [:context, :each].each do |scope|
-    config.before(:context, :quarantine) do |context|
-      skip('In quarantine') unless config.inclusion_filter[:quarantine]
-    end
+    skip_or_run_quarantined_tests(example.metadata.keys, config.inclusion_filter.rules.keys)
   end
 
   config.expect_with :rspec do |expectations|
@@ -66,19 +47,41 @@ RSpec.configure do |config|
   Kernel.srand config.seed
 end
 
-def filters_other_than_quarantine(config)
-  config.inclusion_filter.rules.keys.reject { |key| key == :quarantine }
+# Skip tests in quarantine unless we explicitly focus on them.
+# Skip the entire context if a context is tagged. This avoids running before
+# blocks unnecessarily.
+# If quarantine is focussed, skip tests/contexts that have other metadata
+# unless they're also focussed. This lets us run quarantined tests in a
+# particular category without running tests in other categories.
+# E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged
+# 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests
+# using `--tag quarantine --tag smoke`, without this check we'd end up
+# running that ldap test as well.
+# We could use an exclusion filter, but this way the test report will list
+# the quarantined tests when they're not run so that we're aware of them
+def skip_or_run_quarantined_tests(metadata_keys, filter_keys)
+  included_filters = filters_other_than_quarantine(filter_keys)
+
+  if filter_keys.include?(:quarantine)
+    skip("Only running tests tagged with :quarantine and any of #{included_filters}") unless quarantine_and_optional_other_tag?(metadata_keys, included_filters)
+  else
+    skip('In quarantine') if metadata_keys.include?(:quarantine)
+  end
+end
+
+def filters_other_than_quarantine(filter_keys)
+  filter_keys.reject { |key| key == :quarantine }
 end
 
 # Checks if a test has the 'quarantine' tag and other tags in the inclusion filter.
 #
 # Returns true if
 # - the metadata includes the quarantine tag
-# - and the metadata and inclusion filter both have any other tag
-# - or no other tags are in the inclusion filter
-def quarantine_and_optional_other_tag?(metadata_keys, config)
+#   - and the metadata and inclusion filter both have any other tag
+#   - or no other tags are in the inclusion filter
+def quarantine_and_optional_other_tag?(metadata_keys, included_filters)
   return false unless metadata_keys.include? :quarantine
-  return true if filters_other_than_quarantine(config).empty?
+  return true if included_filters.empty?
 
-  filters_other_than_quarantine(config).any? { |key| metadata_keys.include? key }
+  included_filters.any? { |key| metadata_keys.include? key }
 end
diff --git a/qa/spec/spec_helper_spec.rb b/qa/spec/spec_helper_spec.rb
index ff31999416f..2427999e110 100644
--- a/qa/spec/spec_helper_spec.rb
+++ b/qa/spec/spec_helper_spec.rb
@@ -10,79 +10,79 @@ describe 'rspec config tests' do
         end
       end
 
+      context 'default' do
+        it_behaves_like 'passing tests'
+      end
+
       context 'foo', :foo do
         it_behaves_like 'passing tests'
       end
 
-      context 'default' do
+      context 'quarantine', :quarantine do
+        it_behaves_like 'passing tests'
+      end
+
+      context 'bar quarantine', :bar, :quarantine do
         it_behaves_like 'passing tests'
       end
     end
   end
 
-  context 'default config' do
-    it 'tests are skipped if in quarantine' do
+  context 'with no tags focussed' do
+    before do
       group.run
+    end
 
-      foo_context = group.children.find { |c| c.description == "foo" }
-      foo_examples = foo_context.descendant_filtered_examples
-      expect(foo_examples.count).to eq(2)
-
-      ex = foo_examples.find { |e| e.description == "not in quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+    context 'in a context tagged :foo' do
+      it 'skips tests in quarantine' do
+        context = group.children.find { |c| c.description == "foo" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to eq(2)
 
-      ex = foo_examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('In quarantine')
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
 
-      default_context = group.children.find { |c| c.description == "default" }
-      default_examples = default_context.descendant_filtered_examples
-      expect(default_examples.count).to eq(2)
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+        expect(ex.execution_result.pending_message).to eq('In quarantine')
+      end
+    end
 
-      ex = default_examples.find { |e| e.description == "not in quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+    context 'in an untagged context' do
+      it 'skips tests in quarantine' do
+        context = group.children.find { |c| c.description == "default" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to eq(2)
 
-      ex = default_examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('In quarantine')
-    end
-  end
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
 
-  context "with 'quarantine' tagged" do
-    before do
-      RSpec.configure do |config|
-        config.inclusion_filter = :quarantine
-      end
-    end
-    after do
-      RSpec.configure do |config|
-        config.inclusion_filter.clear
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+        expect(ex.execution_result.pending_message).to eq('In quarantine')
       end
     end
 
-    it "only quarantined tests are run" do
-      group.run
-
-      foo_context = group.children.find { |c| c.description == "foo" }
-      foo_examples = foo_context.descendant_filtered_examples
-      expect(foo_examples.count).to be(1)
-
-      ex = foo_examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+    context 'in a context tagged :quarantine' do
+      it 'skips all tests' do
+        context = group.children.find { |c| c.description == "quarantine" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to eq(2)
 
-      default_context = group.children.find { |c| c.description == "default" }
-      default_examples = default_context.descendant_filtered_examples
-      expect(default_examples.count).to be(1)
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
 
-      ex = default_examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+        expect(ex.execution_result.pending_message).to eq('In quarantine')
+      end
     end
   end
 
-  context "with 'foo' tagged" do
+  context 'with :quarantine focussed' do
     before do
       RSpec.configure do |config|
-        config.inclusion_filter = :foo
+        config.inclusion_filter = :quarantine
       end
 
       group.run
@@ -93,30 +93,50 @@ describe 'rspec config tests' do
       end
     end
 
-    it "tests are not run if not tagged 'foo'" do
-      default_context = group.children.find { |c| c.description == "default" }
-      expect(default_context.descendant_filtered_examples.count).to eq(0)
+    context 'in an untagged context' do
+      it 'only runs quarantined tests' do
+        context = group.children.find { |c| c.description == "default" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(1)
+
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
+      end
     end
 
-    it "tests are skipped if in quarantine" do
-      foo_context = group.children.find { |c| c.description == "foo" }
-      foo_examples = foo_context.descendant_filtered_examples
-      expect(foo_examples.count).to eq(2)
+    context 'in a context tagged :foo' do
+      it 'only runs quarantined tests' do
+        context = group.children.find { |c| c.description == "foo" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(1)
 
-      ex = foo_examples.find { |e| e.description == "not in quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
+      end
+    end
+
+    context 'in a context tagged :quarantine' do
+      it 'runs all tests' do
+        context = group.children.find { |c| c.description == "quarantine" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
+
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
 
-      ex = foo_examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('In quarantine')
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
+      end
     end
   end
 
-  context "with 'quarantine' and 'foo' tagged" do
+  context 'with a non-quarantine tag (:foo) focussed' do
     before do
       RSpec.configure do |config|
-        config.inclusion_filter = { quarantine: true, foo: true }
+        config.inclusion_filter = :foo
       end
+
+      group.run
     end
     after do
       RSpec.configure do |config|
@@ -124,38 +144,43 @@ describe 'rspec config tests' do
       end
     end
 
-    it 'of tests tagged foo, only tests in quarantine run' do
-      group.run
+    context 'in an untagged context' do
+      it 'runs no tests' do
+        context = group.children.find { |c| c.description == "default" }
+        expect(context.descendant_filtered_examples.count).to eq(0)
+      end
+    end
 
-      foo_context = group.children.find { |c| c.description == "foo" }
-      foo_examples = foo_context.descendant_filtered_examples
-      expect(foo_examples.count).to eq(2)
+    context 'in a context tagged :foo' do
+      it 'skips quarantined tests' do
+        context = group.children.find { |c| c.description == "foo" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
 
-      ex = foo_examples.find { |e| e.description == "not in quarantine" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:foo]')
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
 
-      ex = foo_examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+        expect(ex.execution_result.pending_message).to eq('In quarantine')
+      end
     end
 
-    it 'if tests are not tagged they are skipped, even if they are in quarantine' do
-      group.run
-      default_context = group.children.find { |c| c.description == "default" }
-      default_examples = default_context.descendant_filtered_examples
-      expect(default_examples.count).to eq(1)
-
-      ex = default_examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:foo]')
+    context 'in a context tagged :quarantine' do
+      it 'runs no tests' do
+        context = group.children.find { |c| c.description == "quarantine" }
+        expect(context.descendant_filtered_examples.count).to eq(0)
+      end
     end
   end
 
-  context "with 'foo' and 'bar' tagged" do
+  context 'with :quarantine and a non-quarantine tag (:foo) focussed' do
     before do
       RSpec.configure do |config|
-        config.inclusion_filter = { bar: true, foo: true }
+        config.inclusion_filter = { quarantine: true, foo: true }
       end
+
+      group.run
     end
     after do
       RSpec.configure do |config|
@@ -163,67 +188,67 @@ describe 'rspec config tests' do
       end
     end
 
-    it "runs tests tagged either 'foo' or 'bar'" do
-      group = RSpec.describe do
-        example 'foo', :foo do
-        end
-        example 'bar', :bar do
-        end
-        example 'foo and bar', :foo, :bar do
-        end
-      end
-
-      group.run
-      expect(group.examples.count).to eq(3)
+    context 'in an untagged context' do
+      it 'ignores untagged tests and skips tests even if in quarantine' do
+        context = group.children.find { |c| c.description == "default" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to eq(1)
 
-      ex = group.examples.find { |e| e.description == "foo" }
-      expect(ex.execution_result.status).to eq(:passed)
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+      end
+    end
 
-      ex = group.examples.find { |e| e.description == "bar" }
-      expect(ex.execution_result.status).to eq(:passed)
+    context 'in a context tagged :foo' do
+      it 'only runs quarantined tests' do
+        context = group.children.find { |c| c.description == "foo" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
 
-      ex = group.examples.find { |e| e.description == "foo and bar" }
-      expect(ex.execution_result.status).to eq(:passed)
-    end
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
 
-    it "skips quarantined tests tagged 'foo' and/or 'bar'" do
-      group = RSpec.describe do
-        example 'foo in quarantine', :foo, :quarantine do
-        end
-        example 'foo and bar in quarantine', :foo, :bar, :quarantine do
-        end
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
       end
+    end
 
-      group.run
-      expect(group.examples.count).to eq(2)
+    context 'in a context tagged :quarantine' do
+      it 'skips all tests' do
+        context = group.children.find { |c| c.description == "quarantine" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
 
-      ex = group.examples.find { |e| e.description == "foo in quarantine" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('In quarantine')
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
 
-      ex = group.examples.find { |e| e.description == "foo and bar in quarantine" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('In quarantine')
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+      end
     end
 
-    it "ignores quarantined tests not tagged either 'foo' or 'bar'" do
-      group = RSpec.describe do
-        example 'in quarantine', :quarantine do
-        end
-      end
+    context 'in a context tagged :bar and :quarantine' do
+      it 'skips all tests' do
+        context = group.children.find { |c| c.description == "quarantine" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
 
-      group.run
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
 
-      ex = group.examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to be_nil
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+      end
     end
   end
 
-  context "with 'foo' and 'bar' and 'quarantined' tagged" do
+  context 'with :quarantine and multiple non-quarantine tags focussed' do
     before do
       RSpec.configure do |config|
         config.inclusion_filter = { bar: true, foo: true, quarantine: true }
       end
+
+      group.run
     end
     after do
       RSpec.configure do |config|
@@ -231,34 +256,49 @@ describe 'rspec config tests' do
       end
     end
 
-    it "runs tests tagged 'quarantine' and 'foo' or 'bar'" do
-      group = RSpec.describe do
-        example 'foo', :foo do
-        end
-        example 'bar and quarantine', :bar, :quarantine do
-        end
-        example 'foo and bar', :foo, :bar do
-        end
-        example 'foo, bar, and quarantine', :foo, :bar, :quarantine do
-        end
+    context 'in a context tagged :foo' do
+      it 'only runs quarantined tests' do
+        context = group.children.find { |c| c.description == "foo" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
+
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
+
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+        expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
       end
+    end
 
-      group.run
-      expect(group.examples.count).to eq(4)
+    context 'in a context tagged :quarantine' do
+      it 'skips all tests' do
+        context = group.children.find { |c| c.description == "quarantine" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
 
-      ex = group.examples.find { |e| e.description == "foo" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+        expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
 
-      ex = group.examples.find { |e| e.description == "bar and quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+        expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
+      end
+    end
 
-      ex = group.examples.find { |e| e.description == "foo and bar" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
+    context 'in a context tagged :bar and :quarantine' do
+      it 'runs all tests' do
+        context = group.children.find { |c| c.description == "bar quarantine" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
 
-      ex = group.examples.find { |e| e.description == "foo, bar, and quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
+
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
+      end
     end
   end
 end