Skip quarantined tests via filters

Instead of modifying the runner, use config hooks to skip quarantined
tests, and to allow only quarantined tests to be run, if desired.

This way quarantined tests are skipped, not excluded completely, so
they are still included in test results.

2 files changed, 299 insertions, 0 deletions

diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb
index 8e01da01340..53a3aa4c727 100644
--- a/qa/spec/spec_helper.rb
+++ b/qa/spec/spec_helper.rb
@@ -5,6 +5,24 @@ Dir[::File.join(__dir__, 'support', '**', '*.rb')].each { |f| require f }
 RSpec.configure do |config|
   config.before do |example|
     QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug?
+
+    # If quarantine is tagged, skip tests that have other metadata unless
+    # they're also tagged. This lets us run quarantined tests in a particular
+    # category without running tests in other categories.
+    # E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged
+    # 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests
+    # using `--tag quarantine --tag smoke`, without this check we'd end up
+    # running that ldap test as well.
+    if config.inclusion_filter[:quarantine]
+      skip('Running only tagged tests in quarantine') unless quarantine_and_optional_other_tag?(example, config)
+    end
+  end
+
+  config.before(:each, :quarantine) do |example|
+    # Skip tests in quarantine unless we explicitly focus on them
+    # We could use an exclusion filter, but this way the test report will list
+    # the quarantined tests when they're not run so that we're aware of them
+    skip('In quarantine') unless config.inclusion_filter[:quarantine]
   end
 
   config.expect_with :rspec do |expectations|
@@ -22,3 +40,20 @@ RSpec.configure do |config|
   config.order = :random
   Kernel.srand config.seed
 end
+
+# Checks if a test has the 'quarantine' tag and other tags in the inclusion filter.
+#
+# Returns true if
+# - the example metadata includes the quarantine tag
+# - and the metadata and inclusion filter both have any other tag
+# - or no other tags are in the inclusion filter
+def quarantine_and_optional_other_tag?(example, config)
+  return false unless example.metadata.keys.include? :quarantine
+
+  filters_other_than_quarantine = config.inclusion_filter.rules.keys.dup
+  filters_other_than_quarantine.delete :quarantine
+
+  return true if filters_other_than_quarantine.empty?
+
+  filters_other_than_quarantine.any? { |key| example.metadata.keys.include? key }
+end
diff --git a/qa/spec/spec_helper_spec.rb b/qa/spec/spec_helper_spec.rb
new file mode 100644
index 00000000000..5a6a142f021
--- /dev/null
+++ b/qa/spec/spec_helper_spec.rb
@@ -0,0 +1,264 @@
+# frozen_string_literal: true
+
+describe 'rspec config tests' do
+  let(:group) do
+    RSpec.describe do
+      shared_examples 'passing tests' do
+        example 'pass: not in quarantine' do
+        end
+        example 'pass: in quarantine', :quarantine do
+        end
+      end
+
+      context 'foo', :foo do
+        it_behaves_like 'passing tests'
+      end
+
+      context 'default' do
+        it_behaves_like 'passing tests'
+      end
+    end
+  end
+
+  context 'default config' do
+    it 'tests are skipped if in quarantine' do
+      group.run
+
+      foo_context = group.children.find { |c| c.description == "foo" }
+      foo_examples = foo_context.descendant_filtered_examples
+      expect(foo_examples.count).to eq(2)
+
+      ex = foo_examples.find { |e| e.description == "pass: not in quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+
+      ex = foo_examples.find { |e| e.description == "pass: in quarantine" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('In quarantine')
+
+      default_context = group.children.find { |c| c.description == "default" }
+      default_examples = default_context.descendant_filtered_examples
+      expect(default_examples.count).to eq(2)
+
+      ex = default_examples.find { |e| e.description == "pass: not in quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+
+      ex = default_examples.find { |e| e.description == "pass: in quarantine" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('In quarantine')
+    end
+  end
+
+  context "with 'quarantine' tagged" do
+    before do
+      RSpec.configure do |config|
+        config.inclusion_filter = :quarantine
+      end
+    end
+    after do
+      RSpec.configure do |config|
+        config.inclusion_filter.clear
+      end
+    end
+
+    it "only quarantined tests are run" do
+      group.run
+
+      foo_context = group.children.find { |c| c.description == "foo" }
+      foo_examples = foo_context.descendant_filtered_examples
+      expect(foo_examples.count).to be(1)
+
+      ex = foo_examples.find { |e| e.description == "pass: in quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+
+      default_context = group.children.find { |c| c.description == "default" }
+      default_examples = default_context.descendant_filtered_examples
+      expect(default_examples.count).to be(1)
+
+      ex = default_examples.find { |e| e.description == "pass: in quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+    end
+  end
+
+  context "with 'foo' tagged" do
+    before do
+      RSpec.configure do |config|
+        config.inclusion_filter = :foo
+      end
+
+      group.run
+    end
+    after do
+      RSpec.configure do |config|
+        config.inclusion_filter.clear
+      end
+    end
+
+    it "tests are not run if not tagged 'foo'" do
+      default_context = group.children.find { |c| c.description == "default" }
+      expect(default_context.descendant_filtered_examples.count).to eq(0)
+    end
+
+    it "tests are skipped if in quarantine" do
+      foo_context = group.children.find { |c| c.description == "foo" }
+      foo_examples = foo_context.descendant_filtered_examples
+      expect(foo_examples.count).to eq(2)
+
+      ex = foo_examples.find { |e| e.description == "pass: not in quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+
+      ex = foo_examples.find { |e| e.description == "pass: in quarantine" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('In quarantine')
+    end
+  end
+
+  context "with 'quarantine' and 'foo' tagged" do
+    before do
+      RSpec.configure do |config|
+        config.inclusion_filter = { quarantine: true, foo: true }
+      end
+    end
+    after do
+      RSpec.configure do |config|
+        config.inclusion_filter.clear
+      end
+    end
+
+    it 'of tests tagged foo, only tests in quarantine run' do
+      group.run
+
+      foo_context = group.children.find { |c| c.description == "foo" }
+      foo_examples = foo_context.descendant_filtered_examples
+      expect(foo_examples.count).to eq(2)
+
+      ex = foo_examples.find { |e| e.description == "pass: not in quarantine" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('Running only tagged tests in quarantine')
+
+      ex = foo_examples.find { |e| e.description == "pass: in quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+    end
+
+    it 'if tests are not tagged they are skipped, even if they are in quarantine' do
+      group.run
+      default_context = group.children.find { |c| c.description == "default" }
+      default_examples = default_context.descendant_filtered_examples
+      expect(default_examples.count).to eq(1)
+
+      ex = default_examples.find { |e| e.description == "pass: in quarantine" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('Running only tagged tests in quarantine')
+    end
+  end
+
+  context "with 'foo' and 'bar' tagged" do
+    before do
+      RSpec.configure do |config|
+        config.inclusion_filter = { bar: true, foo: true }
+      end
+    end
+    after do
+      RSpec.configure do |config|
+        config.inclusion_filter.clear
+      end
+    end
+
+    it "runs tests tagged either 'foo' or 'bar'" do
+      group = RSpec.describe do
+        example 'foo', :foo do
+        end
+        example 'bar', :bar do
+        end
+        example 'foo and bar', :foo, :bar do
+        end
+      end
+
+      group.run
+      expect(group.examples.count).to eq(3)
+
+      ex = group.examples.find { |e| e.description == "foo" }
+      expect(ex.execution_result.status).to eq(:passed)
+
+      ex = group.examples.find { |e| e.description == "bar" }
+      expect(ex.execution_result.status).to eq(:passed)
+
+      ex = group.examples.find { |e| e.description == "foo and bar" }
+      expect(ex.execution_result.status).to eq(:passed)
+    end
+
+    it "skips quarantined tests tagged 'foo' and/or 'bar'" do
+      group = RSpec.describe do
+        example 'foo in quarantine', :foo, :quarantine do
+        end
+        example 'foo and bar in quarantine', :foo, :bar, :quarantine do
+        end
+      end
+
+      group.run
+      expect(group.examples.count).to eq(2)
+
+      ex = group.examples.find { |e| e.description == "foo in quarantine" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('In quarantine')
+
+      ex = group.examples.find { |e| e.description == "foo and bar in quarantine" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('In quarantine')
+    end
+
+    it "ignores quarantined tests not tagged either 'foo' or 'bar'" do
+      group = RSpec.describe do
+        example 'in quarantine', :quarantine do
+        end
+      end
+
+      group.run
+
+      ex = group.examples.find { |e| e.description == "in quarantine" }
+      expect(ex.execution_result.status).to be_nil
+    end
+  end
+
+  context "with 'foo' and 'bar' and 'quarantined' tagged" do
+    before do
+      RSpec.configure do |config|
+        config.inclusion_filter = { bar: true, foo: true, quarantine: true }
+      end
+    end
+    after do
+      RSpec.configure do |config|
+        config.inclusion_filter.clear
+      end
+    end
+
+    it "runs tests tagged 'quarantine' and 'foo' or 'bar'" do
+      group = RSpec.describe do
+        example 'foo', :foo do
+        end
+        example 'bar and quarantine', :bar, :quarantine do
+        end
+        example 'foo and bar', :foo, :bar do
+        end
+        example 'foo, bar, and quarantine', :foo, :bar, :quarantine do
+        end
+      end
+
+      group.run
+      expect(group.examples.count).to eq(4)
+
+      ex = group.examples.find { |e| e.description == "foo" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('Running only tagged tests in quarantine')
+
+      ex = group.examples.find { |e| e.description == "bar and quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+
+      ex = group.examples.find { |e| e.description == "foo and bar" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('Running only tagged tests in quarantine')
+
+      ex = group.examples.find { |e| e.description == "foo, bar, and quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+    end
+  end
+end