diff options
author | Sanad Liaquat <sliaquat@gitlab.com> | 2019-01-09 06:39:40 +0000 |
---|---|---|
committer | Sanad Liaquat <sliaquat@gitlab.com> | 2019-01-09 06:39:40 +0000 |
commit | 9370ba6fc5dc86fdc6992e5a30dd82aab71f7451 (patch) | |
tree | 186bbcf73a4260765928a7c05beb5259880cbf34 /qa | |
parent | a1b84c0ce7fc2e3f910ff30abe1a01dd2422ffbf (diff) | |
parent | f1b36aa46c68eba573f8e25bad91a0d9e1bf24da (diff) | |
download | gitlab-ce-9370ba6fc5dc86fdc6992e5a30dd82aab71f7451.tar.gz |
Merge branch 'qa-quarantine-tests' into 'master'
Allow tests to be quarantined
Closes gitlab-org/quality/team-tasks#80
See merge request gitlab-org/gitlab-ce!24022
Diffstat (limited to 'qa')
-rw-r--r-- | qa/README.md | 23 | ||||
-rw-r--r-- | qa/spec/spec_helper.rb | 34 | ||||
-rw-r--r-- | qa/spec/spec_helper_spec.rb | 264 |
3 files changed, 320 insertions, 1 deletions
diff --git a/qa/README.md b/qa/README.md index 08ba59e117d..5e32496ea9f 100644 --- a/qa/README.md +++ b/qa/README.md @@ -86,7 +86,7 @@ The environment variable `QA_COOKIES` can be set to send additional cookies on every request. This is necessary on gitlab.com to direct traffic to the canary fleet. To do this set `QA_COOKIES="gitlab_canary=true"`. -To set multiple cookies, separate them with the `;` character, for example: `QA_COOKIES="cookie1=value;cookie2=value2"` +To set multiple cookies, separate them with the `;` character, for example: `QA_COOKIES="cookie1=value;cookie2=value2"` ### Building a Docker image to test @@ -100,3 +100,24 @@ docker build -t gitlab/gitlab-ce-qa:nightly . ``` [GDK]: https://gitlab.com/gitlab-org/gitlab-development-kit/ + +### Quarantined tests + +Tests can be put in quarantine by assigning `:quarantine` metadata. This means +they will be skipped unless run with `--tag quarantine`. This can be used for +tests that are expected to fail while a fix is in progress (similar to how +[`skip` or `pending`](https://relishapp.com/rspec/rspec-core/v/3-8/docs/pending-and-skipped-examples) + can be used). + +``` +bin/qa Test::Instance::All http://localhost --tag quarantine +``` + +If `quarantine` is used with other tags, tests will only be run if they have at +least one of the tags other than `quarantine`. This is different from how RSpec +tags usually work, where all tags are inclusive. + +For example, suppose one test has `:smoke` and `:quarantine` metadata, and +another test has `:ldap` and `:quarantine` metadata. If the tests are run with +`--tag smoke --tag quarantine`, only the first test will run. The test with +`:ldap` will not run even though it also has `:quarantine`. diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb index 8e01da01340..3537ba7c235 100644 --- a/qa/spec/spec_helper.rb +++ b/qa/spec/spec_helper.rb @@ -5,6 +5,24 @@ Dir[::File.join(__dir__, 'support', '**', '*.rb')].each { |f| require f } RSpec.configure do |config| config.before do |example| QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug? + + # If quarantine is tagged, skip tests that have other metadata unless + # they're also tagged. This lets us run quarantined tests in a particular + # category without running tests in other categories. + # E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged + # 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests + # using `--tag quarantine --tag smoke`, without this check we'd end up + # running that ldap test as well. + if config.inclusion_filter[:quarantine] + skip("Running tests tagged with all of #{config.inclusion_filter.rules.keys}") unless quarantine_and_optional_other_tag?(example, config) + end + end + + config.before(:each, :quarantine) do |example| + # Skip tests in quarantine unless we explicitly focus on them + # We could use an exclusion filter, but this way the test report will list + # the quarantined tests when they're not run so that we're aware of them + skip('In quarantine') unless config.inclusion_filter[:quarantine] end config.expect_with :rspec do |expectations| @@ -22,3 +40,19 @@ RSpec.configure do |config| config.order = :random Kernel.srand config.seed end + +# Checks if a test has the 'quarantine' tag and other tags in the inclusion filter. +# +# Returns true if +# - the example metadata includes the quarantine tag +# - and the metadata and inclusion filter both have any other tag +# - or no other tags are in the inclusion filter +def quarantine_and_optional_other_tag?(example, config) + return false unless example.metadata.keys.include? :quarantine + + filters_other_than_quarantine = config.inclusion_filter.rules.keys.reject { |key| key == :quarantine } + + return true if filters_other_than_quarantine.empty? + + filters_other_than_quarantine.any? { |key| example.metadata.keys.include? key } +end diff --git a/qa/spec/spec_helper_spec.rb b/qa/spec/spec_helper_spec.rb new file mode 100644 index 00000000000..f001200fb52 --- /dev/null +++ b/qa/spec/spec_helper_spec.rb @@ -0,0 +1,264 @@ +# frozen_string_literal: true + +describe 'rspec config tests' do + let(:group) do + RSpec.describe do + shared_examples 'passing tests' do + example 'not in quarantine' do + end + example 'in quarantine', :quarantine do + end + end + + context 'foo', :foo do + it_behaves_like 'passing tests' + end + + context 'default' do + it_behaves_like 'passing tests' + end + end + end + + context 'default config' do + it 'tests are skipped if in quarantine' do + group.run + + foo_context = group.children.find { |c| c.description == "foo" } + foo_examples = foo_context.descendant_filtered_examples + expect(foo_examples.count).to eq(2) + + ex = foo_examples.find { |e| e.description == "not in quarantine" } + expect(ex.execution_result.status).to eq(:passed) + + ex = foo_examples.find { |e| e.description == "in quarantine" } + expect(ex.execution_result.status).to eq(:pending) + expect(ex.execution_result.pending_message).to eq('In quarantine') + + default_context = group.children.find { |c| c.description == "default" } + default_examples = default_context.descendant_filtered_examples + expect(default_examples.count).to eq(2) + + ex = default_examples.find { |e| e.description == "not in quarantine" } + expect(ex.execution_result.status).to eq(:passed) + + ex = default_examples.find { |e| e.description == "in quarantine" } + expect(ex.execution_result.status).to eq(:pending) + expect(ex.execution_result.pending_message).to eq('In quarantine') + end + end + + context "with 'quarantine' tagged" do + before do + RSpec.configure do |config| + config.inclusion_filter = :quarantine + end + end + after do + RSpec.configure do |config| + config.inclusion_filter.clear + end + end + + it "only quarantined tests are run" do + group.run + + foo_context = group.children.find { |c| c.description == "foo" } + foo_examples = foo_context.descendant_filtered_examples + expect(foo_examples.count).to be(1) + + ex = foo_examples.find { |e| e.description == "in quarantine" } + expect(ex.execution_result.status).to eq(:passed) + + default_context = group.children.find { |c| c.description == "default" } + default_examples = default_context.descendant_filtered_examples + expect(default_examples.count).to be(1) + + ex = default_examples.find { |e| e.description == "in quarantine" } + expect(ex.execution_result.status).to eq(:passed) + end + end + + context "with 'foo' tagged" do + before do + RSpec.configure do |config| + config.inclusion_filter = :foo + end + + group.run + end + after do + RSpec.configure do |config| + config.inclusion_filter.clear + end + end + + it "tests are not run if not tagged 'foo'" do + default_context = group.children.find { |c| c.description == "default" } + expect(default_context.descendant_filtered_examples.count).to eq(0) + end + + it "tests are skipped if in quarantine" do + foo_context = group.children.find { |c| c.description == "foo" } + foo_examples = foo_context.descendant_filtered_examples + expect(foo_examples.count).to eq(2) + + ex = foo_examples.find { |e| e.description == "not in quarantine" } + expect(ex.execution_result.status).to eq(:passed) + + ex = foo_examples.find { |e| e.description == "in quarantine" } + expect(ex.execution_result.status).to eq(:pending) + expect(ex.execution_result.pending_message).to eq('In quarantine') + end + end + + context "with 'quarantine' and 'foo' tagged" do + before do + RSpec.configure do |config| + config.inclusion_filter = { quarantine: true, foo: true } + end + end + after do + RSpec.configure do |config| + config.inclusion_filter.clear + end + end + + it 'of tests tagged foo, only tests in quarantine run' do + group.run + + foo_context = group.children.find { |c| c.description == "foo" } + foo_examples = foo_context.descendant_filtered_examples + expect(foo_examples.count).to eq(2) + + ex = foo_examples.find { |e| e.description == "not in quarantine" } + expect(ex.execution_result.status).to eq(:pending) + expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:quarantine, :foo]') + + ex = foo_examples.find { |e| e.description == "in quarantine" } + expect(ex.execution_result.status).to eq(:passed) + end + + it 'if tests are not tagged they are skipped, even if they are in quarantine' do + group.run + default_context = group.children.find { |c| c.description == "default" } + default_examples = default_context.descendant_filtered_examples + expect(default_examples.count).to eq(1) + + ex = default_examples.find { |e| e.description == "in quarantine" } + expect(ex.execution_result.status).to eq(:pending) + expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:quarantine, :foo]') + end + end + + context "with 'foo' and 'bar' tagged" do + before do + RSpec.configure do |config| + config.inclusion_filter = { bar: true, foo: true } + end + end + after do + RSpec.configure do |config| + config.inclusion_filter.clear + end + end + + it "runs tests tagged either 'foo' or 'bar'" do + group = RSpec.describe do + example 'foo', :foo do + end + example 'bar', :bar do + end + example 'foo and bar', :foo, :bar do + end + end + + group.run + expect(group.examples.count).to eq(3) + + ex = group.examples.find { |e| e.description == "foo" } + expect(ex.execution_result.status).to eq(:passed) + + ex = group.examples.find { |e| e.description == "bar" } + expect(ex.execution_result.status).to eq(:passed) + + ex = group.examples.find { |e| e.description == "foo and bar" } + expect(ex.execution_result.status).to eq(:passed) + end + + it "skips quarantined tests tagged 'foo' and/or 'bar'" do + group = RSpec.describe do + example 'foo in quarantine', :foo, :quarantine do + end + example 'foo and bar in quarantine', :foo, :bar, :quarantine do + end + end + + group.run + expect(group.examples.count).to eq(2) + + ex = group.examples.find { |e| e.description == "foo in quarantine" } + expect(ex.execution_result.status).to eq(:pending) + expect(ex.execution_result.pending_message).to eq('In quarantine') + + ex = group.examples.find { |e| e.description == "foo and bar in quarantine" } + expect(ex.execution_result.status).to eq(:pending) + expect(ex.execution_result.pending_message).to eq('In quarantine') + end + + it "ignores quarantined tests not tagged either 'foo' or 'bar'" do + group = RSpec.describe do + example 'in quarantine', :quarantine do + end + end + + group.run + + ex = group.examples.find { |e| e.description == "in quarantine" } + expect(ex.execution_result.status).to be_nil + end + end + + context "with 'foo' and 'bar' and 'quarantined' tagged" do + before do + RSpec.configure do |config| + config.inclusion_filter = { bar: true, foo: true, quarantine: true } + end + end + after do + RSpec.configure do |config| + config.inclusion_filter.clear + end + end + + it "runs tests tagged 'quarantine' and 'foo' or 'bar'" do + group = RSpec.describe do + example 'foo', :foo do + end + example 'bar and quarantine', :bar, :quarantine do + end + example 'foo and bar', :foo, :bar do + end + example 'foo, bar, and quarantine', :foo, :bar, :quarantine do + end + end + + group.run + expect(group.examples.count).to eq(4) + + ex = group.examples.find { |e| e.description == "foo" } + expect(ex.execution_result.status).to eq(:pending) + expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:bar, :foo, :quarantine]') + + ex = group.examples.find { |e| e.description == "bar and quarantine" } + expect(ex.execution_result.status).to eq(:passed) + + ex = group.examples.find { |e| e.description == "foo and bar" } + expect(ex.execution_result.status).to eq(:pending) + expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:bar, :foo, :quarantine]') + + ex = group.examples.find { |e| e.description == "foo, bar, and quarantine" } + expect(ex.execution_result.status).to eq(:passed) + end + end +end |