Allow tests to be quarantined

Exclude quarantined tests by default.

2 files changed, 84 insertions, 14 deletions

diff --git a/qa/qa/specs/runner.rb b/qa/qa/specs/runner.rb
index 1bd8101c36d..ebd2ed92df9 100644
--- a/qa/qa/specs/runner.rb
+++ b/qa/qa/specs/runner.rb
@@ -23,6 +23,9 @@ module QA
           args.push(%w[--tag ~orchestrated]) unless (%w[-t --tag] & options).any?
         end
 
+        # Skip quarantined tests by default
+        args.push(%w[--tag ~quarantine]) unless include_quarantine?
+
         args.push(%w[--tag ~skip_signup_disabled]) if QA::Runtime::Env.signup_disabled?
 
         QA::Runtime::Env.supported_features.each_key do |key|
@@ -38,6 +41,10 @@ module QA
           abort if status.nonzero?
         end
       end
+
+      def include_quarantine?
+        tags.include?(:quarantine) || options.include?('quarantine')
+      end
     end
   end
 end
diff --git a/qa/spec/specs/runner_spec.rb b/qa/spec/specs/runner_spec.rb
index 741821ddf8c..8e2b852510a 100644
--- a/qa/spec/specs/runner_spec.rb
+++ b/qa/spec/specs/runner_spec.rb
@@ -6,8 +6,12 @@ describe QA::Specs::Runner do
       allow(QA::Runtime::Browser).to receive(:configure!)
     end
 
-    it 'excludes the orchestrated tag by default' do
-      expect_rspec_runner_arguments(['--tag', '~orchestrated', *described_class::DEFAULT_TEST_PATH_ARGS])
+    it 'excludes the orchestrated and quarantine tags by default' do
+      expect_rspec_runner_arguments([
+        '--tag', '~orchestrated',
+        '--tag', '~quarantine',
+        *described_class::DEFAULT_TEST_PATH_ARGS
+      ])
 
       subject.perform
     end
@@ -16,7 +20,12 @@ describe QA::Specs::Runner do
       subject { described_class.new.tap { |runner| runner.tty = true } }
 
       it 'sets the `--tty` flag' do
-        expect_rspec_runner_arguments(['--tty', '--tag', '~orchestrated', *described_class::DEFAULT_TEST_PATH_ARGS])
+        expect_rspec_runner_arguments([
+          '--tty',
+          '--tag', '~orchestrated',
+          '--tag', '~quarantine',
+          *described_class::DEFAULT_TEST_PATH_ARGS
+        ])
 
         subject.perform
       end
@@ -25,18 +34,54 @@ describe QA::Specs::Runner do
     context 'when tags are set' do
       subject { described_class.new.tap { |runner| runner.tags = %i[orchestrated github] } }
 
-      it 'focuses on the given tags' do
-        expect_rspec_runner_arguments(['--tag', 'orchestrated', '--tag', 'github', *described_class::DEFAULT_TEST_PATH_ARGS])
+      it 'focuses on the given tags, plus the ~quarantine tag' do
+        expect_rspec_runner_arguments([
+          '--tag', 'orchestrated',
+          '--tag', 'github',
+          '--tag', '~quarantine',
+          *described_class::DEFAULT_TEST_PATH_ARGS
+        ])
 
         subject.perform
       end
     end
 
-    context 'when "--tag smoke" is set as options' do
+    context 'when only "--tag quarantine" is set as options' do
+      subject { described_class.new.tap { |runner| runner.options = %w[--tag quarantine] } }
+
+      it 'focuses on the given tags without the default args' do
+        expect_rspec_runner_arguments([
+          '--tag', 'quarantine',
+          *described_class::DEFAULT_TEST_PATH_ARGS
+        ])
+
+        subject.perform
+      end
+    end
+
+    context 'when only "--tag smoke" is set as options' do
       subject { described_class.new.tap { |runner| runner.options = %w[--tag smoke] } }
 
-      it 'focuses on the given tag without excluded the orchestrated tag' do
-        expect_rspec_runner_arguments(['--tag', 'smoke', *described_class::DEFAULT_TEST_PATH_ARGS])
+      it 'focuses on the given tag without the default args, but with the ~quarantine tag' do
+        expect_rspec_runner_arguments([
+          '--tag', '~quarantine',
+          '--tag', 'smoke',
+          *described_class::DEFAULT_TEST_PATH_ARGS
+        ])
+
+        subject.perform
+      end
+    end
+
+    context 'when "--tag smoke" and "--tag quarantine" are set as options' do
+      subject { described_class.new.tap { |runner| runner.options = %w[--tag smoke --tag quarantine] } }
+
+      it 'focuses on the given tags without the default args' do
+        expect_rspec_runner_arguments([
+          '--tag', 'smoke',
+          '--tag', 'quarantine',
+          *described_class::DEFAULT_TEST_PATH_ARGS
+        ])
 
         subject.perform
       end
@@ -45,8 +90,12 @@ describe QA::Specs::Runner do
     context 'when "qa/specs/features/foo" is set as options' do
       subject { described_class.new.tap { |runner| runner.options = %w[qa/specs/features/foo] } }
 
-      it 'passes the given tests path and excludes the orchestrated tag' do
-        expect_rspec_runner_arguments(['--tag', '~orchestrated', 'qa/specs/features/foo'])
+      it 'passes the given tests path and excludes the default args' do
+        expect_rspec_runner_arguments([
+          '--tag', '~orchestrated',
+          '--tag', '~quarantine',
+          'qa/specs/features/foo'
+        ])
 
         subject.perform
       end
@@ -55,8 +104,12 @@ describe QA::Specs::Runner do
     context 'when "-- qa/specs/features/foo" is set as options' do
       subject { described_class.new.tap { |runner| runner.options = %w[-- qa/specs/features/foo] } }
 
-      it 'passes the given tests path and excludes the orchestrated tag' do
-        expect_rspec_runner_arguments(['--tag', '~orchestrated', '--', 'qa/specs/features/foo'])
+      it 'passes the given tests path and excludes the default args' do
+        expect_rspec_runner_arguments([
+          '--tag', '~orchestrated',
+          '--tag', '~quarantine',
+          '--', 'qa/specs/features/foo'
+        ])
 
         subject.perform
       end
@@ -70,7 +123,12 @@ describe QA::Specs::Runner do
       subject { described_class.new }
 
       it 'it includes default args and excludes the skip_signup_disabled tag' do
-        expect_rspec_runner_arguments(['--tag', '~orchestrated', '--tag', '~skip_signup_disabled', *described_class::DEFAULT_TEST_PATH_ARGS])
+        expect_rspec_runner_arguments([
+          '--tag', '~orchestrated',
+          '--tag', '~quarantine',
+          '--tag', '~skip_signup_disabled',
+          *described_class::DEFAULT_TEST_PATH_ARGS
+        ])
 
         subject.perform
       end
@@ -84,7 +142,12 @@ describe QA::Specs::Runner do
       subject { described_class.new }
 
       it 'it includes default args and excludes the requires_git_protocol_v2 tag' do
-        expect_rspec_runner_arguments(['--tag', '~orchestrated', '--tag', '~requires_git_protocol_v2', *described_class::DEFAULT_TEST_PATH_ARGS])
+        expect_rspec_runner_arguments([
+          '--tag', '~orchestrated',
+          '--tag', '~quarantine',
+          '--tag', '~requires_git_protocol_v2',
+          *described_class::DEFAULT_TEST_PATH_ARGS
+        ])
 
         subject.perform
       end