1 files changed, 232 insertions, 141 deletions

diff --git a/qa/spec/spec_helper_spec.rb b/qa/spec/spec_helper_spec.rb
index f001200fb52..27ec1ec80fe 100644
--- a/qa/spec/spec_helper_spec.rb
+++ b/qa/spec/spec_helper_spec.rb
@@ -10,49 +10,98 @@ describe 'rspec config tests' do
         end
       end
 
+      context 'default' do
+        it_behaves_like 'passing tests'
+      end
+
       context 'foo', :foo do
         it_behaves_like 'passing tests'
       end
 
-      context 'default' do
+      context 'quarantine', :quarantine do
+        it_behaves_like 'passing tests'
+      end
+
+      context 'bar quarantine', :bar, :quarantine do
         it_behaves_like 'passing tests'
       end
     end
   end
 
-  context 'default config' do
-    it 'tests are skipped if in quarantine' do
+  let(:group_2) do
+    RSpec.describe do
+      before(:all) do
+        @expectations = [1, 2, 3]
+      end
+
+      example 'not in quarantine' do
+        expect(@expectations.shift).to be(3)
+      end
+
+      example 'in quarantine', :quarantine do
+        expect(@expectations.shift).to be(3)
+      end
+    end
+  end
+
+  context 'with no tags focussed' do
+    before do
       group.run
+    end
 
-      foo_context = group.children.find { |c| c.description == "foo" }
-      foo_examples = foo_context.descendant_filtered_examples
-      expect(foo_examples.count).to eq(2)
+    context 'in a context tagged :foo' do
+      it 'skips tests in quarantine' do
+        context = group.children.find { |c| c.description == "foo" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to eq(2)
 
-      ex = foo_examples.find { |e| e.description == "not in quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
 
-      ex = foo_examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('In quarantine')
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+        expect(ex.execution_result.pending_message).to eq('In quarantine')
+      end
+    end
 
-      default_context = group.children.find { |c| c.description == "default" }
-      default_examples = default_context.descendant_filtered_examples
-      expect(default_examples.count).to eq(2)
+    context 'in an untagged context' do
+      it 'skips tests in quarantine' do
+        context = group.children.find { |c| c.description == "default" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to eq(2)
 
-      ex = default_examples.find { |e| e.description == "not in quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
 
-      ex = default_examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('In quarantine')
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+        expect(ex.execution_result.pending_message).to eq('In quarantine')
+      end
+    end
+
+    context 'in a context tagged :quarantine' do
+      it 'skips all tests' do
+        context = group.children.find { |c| c.description == "quarantine" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to eq(2)
+
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+        expect(ex.execution_result.pending_message).to eq('In quarantine')
+      end
     end
   end
 
-  context "with 'quarantine' tagged" do
+  context 'with :quarantine focussed' do
     before do
       RSpec.configure do |config|
         config.inclusion_filter = :quarantine
       end
+
+      group.run
     end
     after do
       RSpec.configure do |config|
@@ -60,26 +109,44 @@ describe 'rspec config tests' do
       end
     end
 
-    it "only quarantined tests are run" do
-      group.run
+    context 'in an untagged context' do
+      it 'only runs quarantined tests' do
+        context = group.children.find { |c| c.description == "default" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(1)
+
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
+      end
+    end
+
+    context 'in a context tagged :foo' do
+      it 'only runs quarantined tests' do
+        context = group.children.find { |c| c.description == "foo" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(1)
 
-      foo_context = group.children.find { |c| c.description == "foo" }
-      foo_examples = foo_context.descendant_filtered_examples
-      expect(foo_examples.count).to be(1)
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
+      end
+    end
 
-      ex = foo_examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+    context 'in a context tagged :quarantine' do
+      it 'runs all tests' do
+        context = group.children.find { |c| c.description == "quarantine" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
 
-      default_context = group.children.find { |c| c.description == "default" }
-      default_examples = default_context.descendant_filtered_examples
-      expect(default_examples.count).to be(1)
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
 
-      ex = default_examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
+      end
     end
   end
 
-  context "with 'foo' tagged" do
+  context 'with a non-quarantine tag (:foo) focussed' do
     before do
       RSpec.configure do |config|
         config.inclusion_filter = :foo
@@ -93,30 +160,43 @@ describe 'rspec config tests' do
       end
     end
 
-    it "tests are not run if not tagged 'foo'" do
-      default_context = group.children.find { |c| c.description == "default" }
-      expect(default_context.descendant_filtered_examples.count).to eq(0)
+    context 'in an untagged context' do
+      it 'runs no tests' do
+        context = group.children.find { |c| c.description == "default" }
+        expect(context.descendant_filtered_examples.count).to eq(0)
+      end
     end
 
-    it "tests are skipped if in quarantine" do
-      foo_context = group.children.find { |c| c.description == "foo" }
-      foo_examples = foo_context.descendant_filtered_examples
-      expect(foo_examples.count).to eq(2)
+    context 'in a context tagged :foo' do
+      it 'skips quarantined tests' do
+        context = group.children.find { |c| c.description == "foo" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
+
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
 
-      ex = foo_examples.find { |e| e.description == "not in quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+        expect(ex.execution_result.pending_message).to eq('In quarantine')
+      end
+    end
 
-      ex = foo_examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('In quarantine')
+    context 'in a context tagged :quarantine' do
+      it 'runs no tests' do
+        context = group.children.find { |c| c.description == "quarantine" }
+        expect(context.descendant_filtered_examples.count).to eq(0)
+      end
     end
   end
 
-  context "with 'quarantine' and 'foo' tagged" do
+  context 'with :quarantine and a non-quarantine tag (:foo) focussed' do
     before do
       RSpec.configure do |config|
         config.inclusion_filter = { quarantine: true, foo: true }
       end
+
+      group.run
     end
     after do
       RSpec.configure do |config|
@@ -124,38 +204,67 @@ describe 'rspec config tests' do
       end
     end
 
-    it 'of tests tagged foo, only tests in quarantine run' do
-      group.run
+    context 'in an untagged context' do
+      it 'ignores untagged tests and skips tests even if in quarantine' do
+        context = group.children.find { |c| c.description == "default" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to eq(1)
+
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+      end
+    end
 
-      foo_context = group.children.find { |c| c.description == "foo" }
-      foo_examples = foo_context.descendant_filtered_examples
-      expect(foo_examples.count).to eq(2)
+    context 'in a context tagged :foo' do
+      it 'only runs quarantined tests' do
+        context = group.children.find { |c| c.description == "foo" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
 
-      ex = foo_examples.find { |e| e.description == "not in quarantine" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:quarantine, :foo]')
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
 
-      ex = foo_examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+      end
     end
 
-    it 'if tests are not tagged they are skipped, even if they are in quarantine' do
-      group.run
-      default_context = group.children.find { |c| c.description == "default" }
-      default_examples = default_context.descendant_filtered_examples
-      expect(default_examples.count).to eq(1)
+    context 'in a context tagged :quarantine' do
+      it 'skips all tests' do
+        context = group.children.find { |c| c.description == "quarantine" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
+
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+      end
+    end
+
+    context 'in a context tagged :bar and :quarantine' do
+      it 'skips all tests' do
+        context = group.children.find { |c| c.description == "quarantine" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
 
-      ex = default_examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:quarantine, :foo]')
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+      end
     end
   end
 
-  context "with 'foo' and 'bar' tagged" do
+  context 'with :quarantine and multiple non-quarantine tags focussed' do
     before do
       RSpec.configure do |config|
-        config.inclusion_filter = { bar: true, foo: true }
+        config.inclusion_filter = { bar: true, foo: true, quarantine: true }
       end
+
+      group.run
     end
     after do
       RSpec.configure do |config|
@@ -163,102 +272,84 @@ describe 'rspec config tests' do
       end
     end
 
-    it "runs tests tagged either 'foo' or 'bar'" do
-      group = RSpec.describe do
-        example 'foo', :foo do
-        end
-        example 'bar', :bar do
-        end
-        example 'foo and bar', :foo, :bar do
-        end
-      end
-
-      group.run
-      expect(group.examples.count).to eq(3)
-
-      ex = group.examples.find { |e| e.description == "foo" }
-      expect(ex.execution_result.status).to eq(:passed)
+    context 'in a context tagged :foo' do
+      it 'only runs quarantined tests' do
+        context = group.children.find { |c| c.description == "foo" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
 
-      ex = group.examples.find { |e| e.description == "bar" }
-      expect(ex.execution_result.status).to eq(:passed)
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
 
-      ex = group.examples.find { |e| e.description == "foo and bar" }
-      expect(ex.execution_result.status).to eq(:passed)
-    end
-
-    it "skips quarantined tests tagged 'foo' and/or 'bar'" do
-      group = RSpec.describe do
-        example 'foo in quarantine', :foo, :quarantine do
-        end
-        example 'foo and bar in quarantine', :foo, :bar, :quarantine do
-        end
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+        expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
       end
+    end
 
-      group.run
-      expect(group.examples.count).to eq(2)
+    context 'in a context tagged :quarantine' do
+      it 'skips all tests' do
+        context = group.children.find { |c| c.description == "quarantine" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
 
-      ex = group.examples.find { |e| e.description == "foo in quarantine" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('In quarantine')
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+        expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
 
-      ex = group.examples.find { |e| e.description == "foo and bar in quarantine" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('In quarantine')
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:pending)
+        expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
+      end
     end
 
-    it "ignores quarantined tests not tagged either 'foo' or 'bar'" do
-      group = RSpec.describe do
-        example 'in quarantine', :quarantine do
-        end
-      end
+    context 'in a context tagged :bar and :quarantine' do
+      it 'runs all tests' do
+        context = group.children.find { |c| c.description == "bar quarantine" }
+        examples = context.descendant_filtered_examples
+        expect(examples.count).to be(2)
 
-      group.run
+        ex = examples.find { |e| e.description == "in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
 
-      ex = group.examples.find { |e| e.description == "in quarantine" }
-      expect(ex.execution_result.status).to be_nil
+        ex = examples.find { |e| e.description == "not in quarantine" }
+        expect(ex.execution_result.status).to eq(:passed)
+      end
     end
   end
 
-  context "with 'foo' and 'bar' and 'quarantined' tagged" do
-    before do
-      RSpec.configure do |config|
-        config.inclusion_filter = { bar: true, foo: true, quarantine: true }
+  context 'rspec retry' do
+    context 'in an untagged context' do
+      before do
+        group_2.run
       end
-    end
-    after do
-      RSpec.configure do |config|
-        config.inclusion_filter.clear
+
+      it 'should run example :retry times' do
+        examples = group_2.descendant_filtered_examples
+        ex = examples.find { |e| e.description == 'not in quarantine' }
+        expect(ex.execution_result.status).to eq(:passed)
       end
     end
 
-    it "runs tests tagged 'quarantine' and 'foo' or 'bar'" do
-      group = RSpec.describe do
-        example 'foo', :foo do
-        end
-        example 'bar and quarantine', :bar, :quarantine do
-        end
-        example 'foo and bar', :foo, :bar do
-        end
-        example 'foo, bar, and quarantine', :foo, :bar, :quarantine do
+    context 'with :quarantine focussed' do
+      before do
+        RSpec.configure do |config|
+          config.inclusion_filter = :quarantine
         end
+        group_2.run
       end
 
-      group.run
-      expect(group.examples.count).to eq(4)
-
-      ex = group.examples.find { |e| e.description == "foo" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:bar, :foo, :quarantine]')
-
-      ex = group.examples.find { |e| e.description == "bar and quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
-
-      ex = group.examples.find { |e| e.description == "foo and bar" }
-      expect(ex.execution_result.status).to eq(:pending)
-      expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:bar, :foo, :quarantine]')
+      after do
+        RSpec.configure do |config|
+          config.inclusion_filter.clear
+        end
+      end
 
-      ex = group.examples.find { |e| e.description == "foo, bar, and quarantine" }
-      expect(ex.execution_result.status).to eq(:passed)
+      it 'should run example once only' do
+        examples = group_2.descendant_filtered_examples
+        ex = examples.find { |e| e.description == 'in quarantine' }
+        expect(ex.execution_result.status).to eq(:failed)
+      end
     end
   end
 end