diff options
| -rw-r--r-- | qa/qa.rb | 4 | ||||
| -rw-r--r-- | qa/qa/specs/helpers/quarantine.rb | 68 | ||||
| -rw-r--r-- | qa/spec/spec_helper.rb | 47 | ||||
| -rw-r--r-- | qa/spec/spec_helper_spec.rb | 355 | ||||
| -rw-r--r-- | qa/spec/specs/helpers/quarantine_spec.rb | 271 |
5 files changed, 344 insertions, 401 deletions
@@ -342,6 +342,10 @@ module QA module Specs autoload :Config, 'qa/specs/config' autoload :Runner, 'qa/specs/runner' + + module Helpers + autoload :Quarantine, 'qa/specs/helpers/quarantine' + end end ## diff --git a/qa/qa/specs/helpers/quarantine.rb b/qa/qa/specs/helpers/quarantine.rb new file mode 100644 index 00000000000..52cb05fcd13 --- /dev/null +++ b/qa/qa/specs/helpers/quarantine.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'rspec/core' + +module QA::Specs::Helpers + module Quarantine + include RSpec::Core::Pending + + extend self + + def configure_rspec + RSpec.configure do |config| + config.before(:context, :quarantine) do + Quarantine.skip_or_run_quarantined_contexts(config.inclusion_filter.rules, self.class) + end + + config.before do |example| + Quarantine.skip_or_run_quarantined_tests_or_contexts(config.inclusion_filter.rules, example) + end + end + end + + # Skip tests in quarantine unless we explicitly focus on them. + def skip_or_run_quarantined_tests_or_contexts(filters, example) + if filters.key?(:quarantine) + included_filters = filters_other_than_quarantine(filters) + + # If :quarantine is focused, skip the test/context unless its metadata + # includes quarantine and any other filters + # E.g., Suppose a test is tagged :smoke and :quarantine, and another is tagged + # :ldap and :quarantine. If we wanted to run just quarantined smoke tests + # using `--tag quarantine --tag smoke`, without this check we'd end up + # running that ldap test as well because of the :quarantine metadata. + # We could use an exclusion filter, but this way the test report will list + # the quarantined tests when they're not run so that we're aware of them + skip("Only running tests tagged with :quarantine and any of #{included_filters.keys}") if should_skip_when_focused?(example.metadata, included_filters) + else + skip('In quarantine') if example.metadata.key?(:quarantine) + end + end + + # Skip the entire context if a context is quarantined. This avoids running + # before blocks unnecessarily. + def skip_or_run_quarantined_contexts(filters, example) + return unless example.metadata.key?(:quarantine) + + skip_or_run_quarantined_tests_or_contexts(filters, example) + end + + def filters_other_than_quarantine(filter) + filter.reject { |key, _| key == :quarantine } + end + + # Checks if a test or context should be skipped. + # + # Returns true if + # - the metadata does not includes the :quarantine tag + # or if + # - the metadata includes the :quarantine tag + # - and the filter includes other tags that aren't in the metadata + def should_skip_when_focused?(metadata, included_filters) + return true unless metadata.key?(:quarantine) + return false if included_filters.empty? + + (metadata.keys & included_filters.keys).empty? + end + end +end diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb index cbdd6e881b1..be13c3fb683 100644 --- a/qa/spec/spec_helper.rb +++ b/qa/spec/spec_helper.rb @@ -6,16 +6,10 @@ require 'rspec/retry' end RSpec.configure do |config| - config.before(:context) do - if self.class.metadata.keys.include?(:quarantine) - skip_or_run_quarantined_tests(self.class.metadata.keys, config.inclusion_filter.rules.keys) - end - end + QA::Specs::Helpers::Quarantine.configure_rspec config.before do |example| QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug? - - skip_or_run_quarantined_tests(example.metadata.keys, config.inclusion_filter.rules.keys) end config.expect_with :rspec do |expectations| @@ -44,42 +38,3 @@ RSpec.configure do |config| example.run_with_retry retry: retry_times end end - -# Skip tests in quarantine unless we explicitly focus on them. -# Skip the entire context if a context is tagged. This avoids running before -# blocks unnecessarily. -# If quarantine is focussed, skip tests/contexts that have other metadata -# unless they're also focussed. This lets us run quarantined tests in a -# particular category without running tests in other categories. -# E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged -# 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests -# using `--tag quarantine --tag smoke`, without this check we'd end up -# running that ldap test as well. -# We could use an exclusion filter, but this way the test report will list -# the quarantined tests when they're not run so that we're aware of them -def skip_or_run_quarantined_tests(metadata_keys, filter_keys) - included_filters = filters_other_than_quarantine(filter_keys) - - if filter_keys.include?(:quarantine) - skip("Only running tests tagged with :quarantine and any of #{included_filters}") unless quarantine_and_optional_other_tag?(metadata_keys, included_filters) - else - skip('In quarantine') if metadata_keys.include?(:quarantine) - end -end - -def filters_other_than_quarantine(filter_keys) - filter_keys.reject { |key| key == :quarantine } -end - -# Checks if a test has the 'quarantine' tag and other tags in the inclusion filter. -# -# Returns true if -# - the metadata includes the quarantine tag -# - and the metadata and inclusion filter both have any other tag -# - or no other tags are in the inclusion filter -def quarantine_and_optional_other_tag?(metadata_keys, included_filters) - return false unless metadata_keys.include? :quarantine - return true if included_filters.empty? - - included_filters.any? { |key| metadata_keys.include? key } -end diff --git a/qa/spec/spec_helper_spec.rb b/qa/spec/spec_helper_spec.rb deleted file mode 100644 index 27ec1ec80fe..00000000000 --- a/qa/spec/spec_helper_spec.rb +++ /dev/null @@ -1,355 +0,0 @@ -# frozen_string_literal: true - -describe 'rspec config tests' do - let(:group) do - RSpec.describe do - shared_examples 'passing tests' do - example 'not in quarantine' do - end - example 'in quarantine', :quarantine do - end - end - - context 'default' do - it_behaves_like 'passing tests' - end - - context 'foo', :foo do - it_behaves_like 'passing tests' - end - - context 'quarantine', :quarantine do - it_behaves_like 'passing tests' - end - - context 'bar quarantine', :bar, :quarantine do - it_behaves_like 'passing tests' - end - end - end - - let(:group_2) do - RSpec.describe do - before(:all) do - @expectations = [1, 2, 3] - end - - example 'not in quarantine' do - expect(@expectations.shift).to be(3) - end - - example 'in quarantine', :quarantine do - expect(@expectations.shift).to be(3) - end - end - end - - context 'with no tags focussed' do - before do - group.run - end - - context 'in a context tagged :foo' do - it 'skips tests in quarantine' do - context = group.children.find { |c| c.description == "foo" } - examples = context.descendant_filtered_examples - expect(examples.count).to eq(2) - - ex = examples.find { |e| e.description == "not in quarantine" } - expect(ex.execution_result.status).to eq(:passed) - - ex = examples.find { |e| e.description == "in quarantine" } - expect(ex.execution_result.status).to eq(:pending) - expect(ex.execution_result.pending_message).to eq('In quarantine') - end - end - - context 'in an untagged context' do - it 'skips tests in quarantine' do - context = group.children.find { |c| c.description == "default" } - examples = context.descendant_filtered_examples - expect(examples.count).to eq(2) - - ex = examples.find { |e| e.description == "not in quarantine" } - expect(ex.execution_result.status).to eq(:passed) - - ex = examples.find { |e| e.description == "in quarantine" } - expect(ex.execution_result.status).to eq(:pending) - expect(ex.execution_result.pending_message).to eq('In quarantine') - end - end - - context 'in a context tagged :quarantine' do - it 'skips all tests' do - context = group.children.find { |c| c.description == "quarantine" } - examples = context.descendant_filtered_examples - expect(examples.count).to eq(2) - - ex = examples.find { |e| e.description == "not in quarantine" } - expect(ex.execution_result.status).to eq(:pending) - - ex = examples.find { |e| e.description == "in quarantine" } - expect(ex.execution_result.status).to eq(:pending) - expect(ex.execution_result.pending_message).to eq('In quarantine') - end - end - end - - context 'with :quarantine focussed' do - before do - RSpec.configure do |config| - config.inclusion_filter = :quarantine - end - - group.run - end - after do - RSpec.configure do |config| - config.inclusion_filter.clear - end - end - - context 'in an untagged context' do - it 'only runs quarantined tests' do - context = group.children.find { |c| c.description == "default" } - examples = context.descendant_filtered_examples - expect(examples.count).to be(1) - - ex = examples.find { |e| e.description == "in quarantine" } - expect(ex.execution_result.status).to eq(:passed) - end - end - - context 'in a context tagged :foo' do - it 'only runs quarantined tests' do - context = group.children.find { |c| c.description == "foo" } - examples = context.descendant_filtered_examples - expect(examples.count).to be(1) - - ex = examples.find { |e| e.description == "in quarantine" } - expect(ex.execution_result.status).to eq(:passed) - end - end - - context 'in a context tagged :quarantine' do - it 'runs all tests' do - context = group.children.find { |c| c.description == "quarantine" } - examples = context.descendant_filtered_examples - expect(examples.count).to be(2) - - ex = examples.find { |e| e.description == "in quarantine" } - expect(ex.execution_result.status).to eq(:passed) - - ex = examples.find { |e| e.description == "not in quarantine" } - expect(ex.execution_result.status).to eq(:passed) - end - end - end - - context 'with a non-quarantine tag (:foo) focussed' do - before do - RSpec.configure do |config| - config.inclusion_filter = :foo - end - - group.run - end - after do - RSpec.configure do |config| - config.inclusion_filter.clear - end - end - - context 'in an untagged context' do - it 'runs no tests' do - context = group.children.find { |c| c.description == "default" } - expect(context.descendant_filtered_examples.count).to eq(0) - end - end - - context 'in a context tagged :foo' do - it 'skips quarantined tests' do - context = group.children.find { |c| c.description == "foo" } - examples = context.descendant_filtered_examples - expect(examples.count).to be(2) - - ex = examples.find { |e| e.description == "not in quarantine" } - expect(ex.execution_result.status).to eq(:passed) - - ex = examples.find { |e| e.description == "in quarantine" } - expect(ex.execution_result.status).to eq(:pending) - expect(ex.execution_result.pending_message).to eq('In quarantine') - end - end - - context 'in a context tagged :quarantine' do - it 'runs no tests' do - context = group.children.find { |c| c.description == "quarantine" } - expect(context.descendant_filtered_examples.count).to eq(0) - end - end - end - - context 'with :quarantine and a non-quarantine tag (:foo) focussed' do - before do - RSpec.configure do |config| - config.inclusion_filter = { quarantine: true, foo: true } - end - - group.run - end - after do - RSpec.configure do |config| - config.inclusion_filter.clear - end - end - - context 'in an untagged context' do - it 'ignores untagged tests and skips tests even if in quarantine' do - context = group.children.find { |c| c.description == "default" } - examples = context.descendant_filtered_examples - expect(examples.count).to eq(1) - - ex = examples.find { |e| e.description == "in quarantine" } - expect(ex.execution_result.status).to eq(:pending) - end - end - - context 'in a context tagged :foo' do - it 'only runs quarantined tests' do - context = group.children.find { |c| c.description == "foo" } - examples = context.descendant_filtered_examples - expect(examples.count).to be(2) - - ex = examples.find { |e| e.description == "in quarantine" } - expect(ex.execution_result.status).to eq(:passed) - - ex = examples.find { |e| e.description == "not in quarantine" } - expect(ex.execution_result.status).to eq(:pending) - end - end - - context 'in a context tagged :quarantine' do - it 'skips all tests' do - context = group.children.find { |c| c.description == "quarantine" } - examples = context.descendant_filtered_examples - expect(examples.count).to be(2) - - ex = examples.find { |e| e.description == "in quarantine" } - expect(ex.execution_result.status).to eq(:pending) - - ex = examples.find { |e| e.description == "not in quarantine" } - expect(ex.execution_result.status).to eq(:pending) - end - end - - context 'in a context tagged :bar and :quarantine' do - it 'skips all tests' do - context = group.children.find { |c| c.description == "quarantine" } - examples = context.descendant_filtered_examples - expect(examples.count).to be(2) - - ex = examples.find { |e| e.description == "in quarantine" } - expect(ex.execution_result.status).to eq(:pending) - - ex = examples.find { |e| e.description == "not in quarantine" } - expect(ex.execution_result.status).to eq(:pending) - end - end - end - - context 'with :quarantine and multiple non-quarantine tags focussed' do - before do - RSpec.configure do |config| - config.inclusion_filter = { bar: true, foo: true, quarantine: true } - end - - group.run - end - after do - RSpec.configure do |config| - config.inclusion_filter.clear - end - end - - context 'in a context tagged :foo' do - it 'only runs quarantined tests' do - context = group.children.find { |c| c.description == "foo" } - examples = context.descendant_filtered_examples - expect(examples.count).to be(2) - - ex = examples.find { |e| e.description == "in quarantine" } - expect(ex.execution_result.status).to eq(:passed) - - ex = examples.find { |e| e.description == "not in quarantine" } - expect(ex.execution_result.status).to eq(:pending) - expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]') - end - end - - context 'in a context tagged :quarantine' do - it 'skips all tests' do - context = group.children.find { |c| c.description == "quarantine" } - examples = context.descendant_filtered_examples - expect(examples.count).to be(2) - - ex = examples.find { |e| e.description == "in quarantine" } - expect(ex.execution_result.status).to eq(:pending) - expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]') - - ex = examples.find { |e| e.description == "not in quarantine" } - expect(ex.execution_result.status).to eq(:pending) - expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]') - end - end - - context 'in a context tagged :bar and :quarantine' do - it 'runs all tests' do - context = group.children.find { |c| c.description == "bar quarantine" } - examples = context.descendant_filtered_examples - expect(examples.count).to be(2) - - ex = examples.find { |e| e.description == "in quarantine" } - expect(ex.execution_result.status).to eq(:passed) - - ex = examples.find { |e| e.description == "not in quarantine" } - expect(ex.execution_result.status).to eq(:passed) - end - end - end - - context 'rspec retry' do - context 'in an untagged context' do - before do - group_2.run - end - - it 'should run example :retry times' do - examples = group_2.descendant_filtered_examples - ex = examples.find { |e| e.description == 'not in quarantine' } - expect(ex.execution_result.status).to eq(:passed) - end - end - - context 'with :quarantine focussed' do - before do - RSpec.configure do |config| - config.inclusion_filter = :quarantine - end - group_2.run - end - - after do - RSpec.configure do |config| - config.inclusion_filter.clear - end - end - - it 'should run example once only' do - examples = group_2.descendant_filtered_examples - ex = examples.find { |e| e.description == 'in quarantine' } - expect(ex.execution_result.status).to eq(:failed) - end - end - end -end diff --git a/qa/spec/specs/helpers/quarantine_spec.rb b/qa/spec/specs/helpers/quarantine_spec.rb new file mode 100644 index 00000000000..78beda39b5e --- /dev/null +++ b/qa/spec/specs/helpers/quarantine_spec.rb @@ -0,0 +1,271 @@ +# frozen_string_literal: true + +require 'rspec/core/sandbox' + +# We need a reporter for internal tests that's different from the reporter for +# external tests otherwise the results will be mixed up. We don't care about +# most reporting, but we do want to know if a test fails +class RaiseOnFailuresReporter < RSpec::Core::NullReporter + def self.example_failed(example) + raise example.exception + end +end + +# We use an example group wrapper to prevent the state of internal tests +# expanding into the global state +# See: https://github.com/rspec/rspec-core/issues/2603 +def describe_successfully(*args, &describe_body) + example_group = RSpec.describe(*args, &describe_body) + ran_successfully = example_group.run RaiseOnFailuresReporter + expect(ran_successfully).to eq true + example_group +end + +RSpec.configure do |c| + c.around do |ex| + RSpec::Core::Sandbox.sandboxed do |config| + # If there is an example-within-an-example, we want to make sure the inner example + # does not get a reference to the outer example (the real spec) if it calls + # something like `pending` + config.before(:context) { RSpec.current_example = nil } + + config.color_mode = :off + + # Load airborne again to avoid "undefined method `match_expected_default?'" errors + # that happen because a hook calls a method added via a custom RSpec setting + # that is removed when the RSpec configuration is sandboxed. + # If this needs to be changed (e.g., to load other libraries as well), see + # this discussion for alternative solutions: + # https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/25223#note_143392053 + load 'airborne.rb' + + ex.run + end + end +end + +describe QA::Specs::Helpers::Quarantine do + describe '.skip_or_run_quarantined_contexts' do + context 'with no tag focused' do + before do + described_class.configure_rspec + end + + it 'skips before hooks of quarantined contexts' do + executed_hooks = [] + + group = describe_successfully('quarantine', :quarantine) do + before(:all) do + executed_hooks << :before_all + end + before do + executed_hooks << :before + end + example {} + end + + expect(executed_hooks).to eq [] + expect(group.descendant_filtered_examples.first.execution_result.status).to eq(:pending) + expect(group.descendant_filtered_examples.first.execution_result.pending_message) + .to eq('In quarantine') + end + + it 'executes before hooks of non-quarantined contexts' do + executed_hooks = [] + + group = describe_successfully do + before(:all) do + executed_hooks << :before_all + end + before do + executed_hooks << :before + end + example {} + end + + expect(executed_hooks).to eq [:before_all, :before] + expect(group.descendant_filtered_examples.first.execution_result.status).to eq(:passed) + end + end + + context 'with :quarantine focused' do + before do + described_class.configure_rspec + RSpec.configure do |c| + c.filter_run :quarantine + end + end + + it 'executes before hooks of quarantined contexts' do + executed_hooks = [] + + group = describe_successfully('quarantine', :quarantine) do + before(:all) do + executed_hooks << :before_all + end + before do + executed_hooks << :before + end + example {} + end + + expect(executed_hooks).to eq [:before_all, :before] + expect(group.descendant_filtered_examples.first.execution_result.status).to eq(:passed) + end + + it 'skips before hooks of non-quarantined contexts' do + executed_hooks = [] + + group = describe_successfully do + before(:all) do + executed_hooks << :before_all + end + before do + executed_hooks << :before + end + example {} + end + + expect(executed_hooks).to eq [] + expect(group.descendant_filtered_examples.first).to be_nil + end + end + end + + describe '.skip_or_run_quarantined_tests' do + context 'with no tag focused' do + before do + described_class.configure_rspec + end + + it 'skips quarantined tests' do + group = describe_successfully do + it('is pending', :quarantine) {} + end + + expect(group.examples.first.execution_result.status).to eq(:pending) + expect(group.examples.first.execution_result.pending_message) + .to eq('In quarantine') + end + + it 'executes non-quarantined tests' do + group = describe_successfully do + example {} + end + + expect(group.examples.first.execution_result.status).to eq(:passed) + end + end + + context 'with :quarantine focused' do + before do + described_class.configure_rspec + RSpec.configure do |c| + c.filter_run :quarantine + end + end + + it 'executes quarantined tests' do + group = describe_successfully do + it('passes', :quarantine) {} + end + + expect(group.examples.first.execution_result.status).to eq(:passed) + end + + it 'ignores non-quarantined tests' do + group = describe_successfully do + example {} + end + + expect(group.examples.first.execution_result.status).to be_nil + end + end + + context 'with a non-quarantine tag focused' do + before do + described_class.configure_rspec + RSpec.configure do |c| + c.filter_run :foo + end + end + + it 'ignores non-quarantined non-focused tests' do + group = describe_successfully do + example {} + end + + expect(group.examples.first.execution_result.status).to be_nil + end + + it 'executes non-quarantined focused tests' do + group = describe_successfully do + it('passes', :foo) {} + end + + expect(group.examples.first.execution_result.status).to be(:passed) + end + + it 'ignores quarantined tests' do + group = describe_successfully do + it('is ignored', :quarantine) {} + end + + expect(group.examples.first.execution_result.status).to be_nil + end + + it 'skips quarantined focused tests' do + group = describe_successfully do + it('is pending', :quarantine, :foo) {} + end + + expect(group.examples.first.execution_result.status).to be(:pending) + expect(group.examples.first.execution_result.pending_message) + .to eq('In quarantine') + end + end + + context 'with :quarantine and non-quarantine tags focused' do + before do + described_class.configure_rspec + RSpec.configure do |c| + c.filter_run :foo, :bar, :quarantine + end + end + + it 'ignores non-quarantined non-focused tests' do + group = describe_successfully do + example {} + end + + expect(group.examples.first.execution_result.status).to be_nil + end + + it 'skips non-quarantined focused tests' do + group = describe_successfully do + it('is pending', :foo) {} + end + + expect(group.examples.first.execution_result.status).to be(:pending) + expect(group.examples.first.execution_result.pending_message) + .to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]') + end + + it 'skips quarantined non-focused tests' do + group = describe_successfully do + it('is pending', :quarantine) {} + end + + expect(group.examples.first.execution_result.status).to be(:pending) + end + + it 'executes quarantined focused tests' do + group = describe_successfully do + it('passes', :quarantine, :foo) {} + end + + expect(group.examples.first.execution_result.status).to be(:passed) + end + end + end +end |