diff options
Diffstat (limited to 'qa/spec')
| -rw-r--r-- | qa/spec/runtime/env_spec.rb | 52 | ||||
| -rw-r--r-- | qa/spec/scenario/test/integration/object_storage_spec.rb | 9 | ||||
| -rw-r--r-- | qa/spec/spec_helper.rb | 1 | ||||
| -rw-r--r-- | qa/spec/specs/helpers/context_selector_spec.rb | 294 | ||||
| -rw-r--r-- | qa/spec/specs/helpers/quarantine_spec.rb | 175 | ||||
| -rw-r--r-- | qa/spec/support/helpers/stub_env.rb | 2 |
6 files changed, 297 insertions, 236 deletions
diff --git a/qa/spec/runtime/env_spec.rb b/qa/spec/runtime/env_spec.rb index 5a98721466f..8218ab428b0 100644 --- a/qa/spec/runtime/env_spec.rb +++ b/qa/spec/runtime/env_spec.rb @@ -341,56 +341,4 @@ RSpec.describe QA::Runtime::Env do end end end - - describe '.context_matches?' do - it 'returns true when url has .com' do - QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com") - - expect(described_class.dot_com?).to be_truthy - end - - it 'returns false when url does not have .com' do - QA::Runtime::Scenario.define(:gitlab_address, "https://gitlab.test") - - expect(described_class.dot_com?).to be_falsey - end - - context 'with arguments' do - it 'returns true when :subdomain is set' do - QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com") - - expect(described_class.dot_com?(subdomain: :staging)).to be_truthy - end - - it 'matches multiple subdomains' do - QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com") - - expect(described_class.context_matches?(subdomain: [:release, :staging])).to be_truthy - expect(described_class.context_matches?(:production, subdomain: [:release, :staging])).to be_truthy - end - - it 'matches :production' do - QA::Runtime::Scenario.define(:gitlab_address, "https://gitlab.com/") - - expect(described_class.context_matches?(:production)).to be_truthy - end - - it 'doesnt match with mismatching switches' do - QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.test') - - aggregate_failures do - expect(described_class.context_matches?(tld: '.net')).to be_falsey - expect(described_class.context_matches?(:production)).to be_falsey - expect(described_class.context_matches?(subdomain: [:staging])).to be_falsey - expect(described_class.context_matches?(domain: 'example')).to be_falsey - end - end - end - - it 'returns false for mismatching' do - QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com") - - expect(described_class.context_matches?(:production)).to be_falsey - end - end end diff --git a/qa/spec/scenario/test/integration/object_storage_spec.rb b/qa/spec/scenario/test/integration/object_storage_spec.rb deleted file mode 100644 index 235dd495687..00000000000 --- a/qa/spec/scenario/test/integration/object_storage_spec.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe QA::Scenario::Test::Integration::ObjectStorage do - describe '#perform' do - it_behaves_like 'a QA scenario class' do - let(:tags) { [:object_storage] } - end - end -end diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb index 16d86ef6ed2..631ebf65893 100644 --- a/qa/spec/spec_helper.rb +++ b/qa/spec/spec_helper.rb @@ -22,6 +22,7 @@ RSpec.configure do |config| config.include ::Matchers QA::Specs::Helpers::Quarantine.configure_rspec + QA::Specs::Helpers::ContextSelector.configure_rspec config.before do |example| QA::Runtime::Logger.debug("\nStarting test: #{example.full_description}\n") diff --git a/qa/spec/specs/helpers/context_selector_spec.rb b/qa/spec/specs/helpers/context_selector_spec.rb new file mode 100644 index 00000000000..16b6c6601b1 --- /dev/null +++ b/qa/spec/specs/helpers/context_selector_spec.rb @@ -0,0 +1,294 @@ +# frozen_string_literal: true + +require 'rspec/core/sandbox' + +RSpec.configure do |c| + c.around do |ex| + RSpec::Core::Sandbox.sandboxed do |config| + # If there is an example-within-an-example, we want to make sure the inner example + # does not get a reference to the outer example (the real spec) if it calls + # something like `pending` + config.before(:context) { RSpec.current_example = nil } + + config.color_mode = :off + + ex.run + end + end +end + +RSpec.describe QA::Specs::Helpers::ContextSelector do + include Helpers::StubENV + include QA::Specs::Helpers::RSpec + + before do + QA::Runtime::Scenario.define(:gitlab_address, 'https://staging.gitlab.com') + described_class.configure_rspec + end + + describe '.context_matches?' do + it 'returns true when url has .com' do + QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com") + + expect(described_class.dot_com?).to be_truthy + end + + it 'returns false when url does not have .com' do + QA::Runtime::Scenario.define(:gitlab_address, "https://gitlab.test") + + expect(described_class.dot_com?).to be_falsey + end + + context 'with arguments' do + it 'returns true when :subdomain is set' do + QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com") + + expect(described_class.dot_com?(subdomain: :staging)).to be_truthy + end + + it 'matches multiple subdomains' do + QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com") + + expect(described_class.context_matches?(subdomain: [:release, :staging])).to be_truthy + expect(described_class.context_matches?(:production, subdomain: [:release, :staging])).to be_truthy + end + + it 'matches :production' do + QA::Runtime::Scenario.define(:gitlab_address, "https://gitlab.com/") + + expect(described_class.context_matches?(:production)).to be_truthy + end + + it 'doesnt match with mismatching switches' do + QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.test') + + aggregate_failures do + expect(described_class.context_matches?(tld: '.net')).to be_falsey + expect(described_class.context_matches?(:production)).to be_falsey + expect(described_class.context_matches?(subdomain: [:staging])).to be_falsey + expect(described_class.context_matches?(domain: 'example')).to be_falsey + end + end + end + + it 'returns false for mismatching' do + QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com") + + expect(described_class.context_matches?(:production)).to be_falsey + end + end + + describe 'description and context blocks' do + context 'with environment set' do + it 'can apply to contexts or descriptions' do + group = describe_successfully 'Runs in staging', only: { subdomain: :staging } do + it('runs in staging') {} + end + + expect(group.examples[0].execution_result.status).to eq(:passed) + end + end + + context 'with different environment set' do + before do + QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.com') + described_class.configure_rspec + end + + it 'does not run against production' do + group = describe_successfully 'Runs in staging', :something, only: { subdomain: :staging } do + it('runs in staging') {} + end + + expect(group.examples[0].execution_result.status).to eq(:pending) + end + end + end + + it 'runs only in staging' do + group = describe_successfully do + it('runs in staging', only: { subdomain: :staging }) {} + it('doesnt run in staging', only: :production) {} + it('runs in staging also', only: { subdomain: %i[release staging] }) {} + it('runs in any env') {} + end + + expect(group.examples[0].execution_result.status).to eq(:passed) + expect(group.examples[1].execution_result.status).to eq(:pending) + expect(group.examples[2].execution_result.status).to eq(:passed) + expect(group.examples[3].execution_result.status).to eq(:passed) + end + + context 'custom env' do + before do + QA::Runtime::Scenario.define(:gitlab_address, 'https://release.gitlab.net') + end + + it 'runs on a custom environment' do + group = describe_successfully do + it('runs on release gitlab net', only: { tld: '.net', subdomain: :release, domain: 'gitlab' }) {} + it('does not run on release', only: :production) {} + end + + expect(group.examples.first.execution_result.status).to eq(:passed) + expect(group.examples.last.execution_result.status).to eq(:pending) + end + end + + context 'production' do + before do + QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.com/') + end + + it 'runs on production' do + group = describe_successfully do + it('runs on prod', only: :production) {} + it('does not run in prod', only: { subdomain: :staging }) {} + it('runs in prod and staging', only: { subdomain: /(staging.)?/, domain: 'gitlab' }) {} + end + + expect(group.examples[0].execution_result.status).to eq(:passed) + expect(group.examples[1].execution_result.status).to eq(:pending) + expect(group.examples[2].execution_result.status).to eq(:passed) + end + end + + it 'outputs a message for invalid environments' do + group = describe_successfully do + it('will skip', only: :production) {} + end + + expect(group.examples.first.execution_result.pending_message).to match(/[Tt]est.*not compatible.*environment/) + end + + context 'with pipeline constraints' do + context 'without CI_PROJECT_NAME set' do + before do + stub_env('CI_PROJECT_NAME', nil) + described_class.configure_rspec + end + + it 'runs on any pipeline' do + group = describe_successfully do + it('runs given a single named pipeline', only: { pipeline: :nightly }) {} + it('runs given an array of pipelines', only: { pipeline: [:canary, :not_nightly] }) {} + end + + aggregate_failures do + expect(group.examples[0].execution_result.status).to eq(:passed) + expect(group.examples[1].execution_result.status).to eq(:passed) + end + end + end + + context 'when a pipeline triggered from the default branch runs in gitlab-qa' do + before do + stub_env('CI_PROJECT_NAME', 'gitlab-qa') + described_class.configure_rspec + end + + it 'runs on default branch pipelines' do + group = describe_successfully do + it('runs on master pipeline given a single pipeline', only: { pipeline: :master }) {} + it('runs in master given an array of pipelines', only: { pipeline: [:canary, :master] }) {} + it('does not run in non-default pipelines', only: { pipeline: [:nightly, :not_nightly, :not_master] }) {} + end + + aggregate_failures do + expect(group.examples[0].execution_result.status).to eq(:passed) + expect(group.examples[1].execution_result.status).to eq(:passed) + expect(group.examples[2].execution_result.status).to eq(:pending) + end + end + end + + context 'with CI_PROJECT_NAME set' do + before do + stub_env('CI_PROJECT_NAME', 'NIGHTLY') + described_class.configure_rspec + end + + it 'runs on designated pipeline' do + group = describe_successfully do + it('runs on nightly', only: { pipeline: :nightly }) {} + it('does not run in not_nightly', only: { pipeline: :not_nightly }) {} + it('runs on nightly given an array', only: { pipeline: [:canary, :nightly] }) {} + it('does not run in not_nightly given an array', only: { pipeline: [:not_nightly, :canary] }) {} + end + + aggregate_failures do + expect(group.examples[0].execution_result.status).to eq(:passed) + expect(group.examples[1].execution_result.status).to eq(:pending) + expect(group.examples[2].execution_result.status).to eq(:passed) + expect(group.examples[3].execution_result.status).to eq(:pending) + end + end + end + end + + context 'when excluding contexts' do + context 'with job constraints' do + context 'without CI_JOB_NAME set' do + before do + stub_env('CI_JOB_NAME', nil) + described_class.configure_rspec + end + + it 'runs in any job' do + group = describe_successfully do + it('runs given a single named job', exclude: { job: 'ee:instance-image' }) {} + it('runs given a single regex pattern', exclude: { job: '.*:instance-image' }) {} + it('runs given an array of jobs', exclude: { job: %w[ee:instance-image qa-schedules-browser_ui-3_create] }) {} + it('runs given an array of regex patterns', exclude: { job: %w[ee:.* qa-schedules-browser_ui.*] }) {} + it('runs given a mix of strings and regex patterns', exclude: { job: %w[ee:instance-image qa-schedules-browser_ui.*] }) {} + end + + aggregate_failures do + group.examples.each do |example| + expect(example.execution_result.status).to eq(:passed) + end + end + end + end + + context 'with CI_JOB_NAME set' do + before do + stub_env('CI_JOB_NAME', 'ee:instance-image') + described_class.configure_rspec + end + + it 'does not run in the specified job' do + group = describe_successfully do + it('skips given a single named job', exclude: { job: 'ee:instance-image' }) {} + it('skips given a single regex pattern', exclude: { job: '.*:instance-image' }) {} + it('skips given an array of jobs', exclude: { job: %w[ee:instance-image qa-schedules-browser_ui-3_create] }) {} + it('skips given an array of regex patterns', exclude: { job: %w[ee:.* qa-schedules-browser_ui.*] }) {} + it('skips given a mix of strings and regex patterns', exclude: { job: %w[ee:instance-image qa-schedules-browser_ui.*] }) {} + end + + aggregate_failures do + group.examples.each do |example| + expect(example.execution_result.status).to eq(:pending) + end + end + end + + it 'runs in jobs that do not match' do + group = describe_successfully do + it('runs given a single named job', exclude: { job: 'ce:instance-image' }) {} + it('runs given a single regex pattern', exclude: { job: '.*:instance-image-quarantine' }) {} + it('runs given an array of jobs', exclude: { job: %w[ce:instance-image qa-schedules-browser_ui-3_create] }) {} + it('runs given an array of regex patterns', exclude: { job: %w[ce:.* qa-schedules-browser_ui.*] }) {} + it('runs given a mix of strings and regex patterns', exclude: { job: %w[ce:instance-image qa-schedules-browser_ui.*] }) {} + end + + aggregate_failures do + group.examples.each do |example| + expect(example.execution_result.status).to eq(:passed) + end + end + end + end + end + end +end diff --git a/qa/spec/specs/helpers/quarantine_spec.rb b/qa/spec/specs/helpers/quarantine_spec.rb index 694c320ce3d..45754a09b17 100644 --- a/qa/spec/specs/helpers/quarantine_spec.rb +++ b/qa/spec/specs/helpers/quarantine_spec.rb @@ -2,25 +2,6 @@ require 'rspec/core/sandbox' -# We need a reporter for internal tests that's different from the reporter for -# external tests otherwise the results will be mixed up. We don't care about -# most reporting, but we do want to know if a test fails -class RaiseOnFailuresReporter < RSpec::Core::NullReporter - def self.example_failed(example) - raise example.exception - end -end - -# We use an example group wrapper to prevent the state of internal tests -# expanding into the global state -# See: https://github.com/rspec/rspec-core/issues/2603 -def describe_successfully(*args, &describe_body) - example_group = RSpec.describe(*args, &describe_body) - ran_successfully = example_group.run RaiseOnFailuresReporter - expect(ran_successfully).to eq true - example_group -end - RSpec.configure do |c| c.around do |ex| RSpec::Core::Sandbox.sandboxed do |config| @@ -38,6 +19,7 @@ end RSpec.describe QA::Specs::Helpers::Quarantine do include Helpers::StubENV + include QA::Specs::Helpers::RSpec describe '.skip_or_run_quarantined_contexts' do context 'with no tag focused' do @@ -336,159 +318,4 @@ RSpec.describe QA::Specs::Helpers::Quarantine do end end end - - describe 'running against specific environments or pipelines' do - before do - QA::Runtime::Scenario.define(:gitlab_address, 'https://staging.gitlab.com') - described_class.configure_rspec - end - - describe 'description and context blocks' do - context 'with environment set' do - it 'can apply to contexts or descriptions' do - group = describe_successfully 'Runs in staging', only: { subdomain: :staging } do - it('runs in staging') {} - end - - expect(group.examples[0].execution_result.status).to eq(:passed) - end - end - - context 'with different environment set' do - before do - QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.com') - described_class.configure_rspec - end - - it 'does not run against production' do - group = describe_successfully 'Runs in staging', :something, only: { subdomain: :staging } do - it('runs in staging') {} - end - - expect(group.examples[0].execution_result.status).to eq(:pending) - end - end - end - - it 'runs only in staging' do - group = describe_successfully do - it('runs in staging', only: { subdomain: :staging }) {} - it('doesnt run in staging', only: :production) {} - it('runs in staging also', only: { subdomain: %i[release staging] }) {} - it('runs in any env') {} - end - - expect(group.examples[0].execution_result.status).to eq(:passed) - expect(group.examples[1].execution_result.status).to eq(:pending) - expect(group.examples[2].execution_result.status).to eq(:passed) - expect(group.examples[3].execution_result.status).to eq(:passed) - end - - context 'custom env' do - before do - QA::Runtime::Scenario.define(:gitlab_address, 'https://release.gitlab.net') - end - - it 'runs on a custom environment' do - group = describe_successfully do - it('runs on release gitlab net', only: { tld: '.net', subdomain: :release, domain: 'gitlab' }) {} - it('does not run on release', only: :production) {} - end - - expect(group.examples.first.execution_result.status).to eq(:passed) - expect(group.examples.last.execution_result.status).to eq(:pending) - end - end - - context 'production' do - before do - QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.com/') - end - - it 'runs on production' do - group = describe_successfully do - it('runs on prod', only: :production) {} - it('does not run in prod', only: { subdomain: :staging }) {} - it('runs in prod and staging', only: { subdomain: /(staging.)?/, domain: 'gitlab' }) {} - end - - expect(group.examples[0].execution_result.status).to eq(:passed) - expect(group.examples[1].execution_result.status).to eq(:pending) - expect(group.examples[2].execution_result.status).to eq(:passed) - end - end - - it 'outputs a message for invalid environments' do - group = describe_successfully do - it('will skip', only: :production) {} - end - - expect(group.examples.first.execution_result.pending_message).to match(/[Tt]est.*not compatible.*environment/) - end - - context 'with pipeline constraints' do - context 'without CI_PROJECT_NAME set' do - before do - stub_env('CI_PROJECT_NAME', nil) - described_class.configure_rspec - end - - it 'runs on any pipeline' do - group = describe_successfully do - it('runs given a single named pipeline', only: { pipeline: :nightly }) {} - it('runs given an array of pipelines', only: { pipeline: [:canary, :not_nightly] }) {} - end - - aggregate_failures do - expect(group.examples[0].execution_result.status).to eq(:passed) - expect(group.examples[1].execution_result.status).to eq(:passed) - end - end - end - - context 'when a pipeline triggered from the default branch runs in gitlab-qa' do - before do - stub_env('CI_PROJECT_NAME', 'gitlab-qa') - described_class.configure_rspec - end - - it 'runs on default branch pipelines' do - group = describe_successfully do - it('runs on master pipeline given a single pipeline', only: { pipeline: :master }) {} - it('runs in master given an array of pipelines', only: { pipeline: [:canary, :master] }) {} - it('does not run in non-default pipelines', only: { pipeline: [:nightly, :not_nightly, :not_master] }) {} - end - - aggregate_failures do - expect(group.examples[0].execution_result.status).to eq(:passed) - expect(group.examples[1].execution_result.status).to eq(:passed) - expect(group.examples[2].execution_result.status).to eq(:pending) - end - end - end - - context 'with CI_PROJECT_NAME set' do - before do - stub_env('CI_PROJECT_NAME', 'NIGHTLY') - described_class.configure_rspec - end - - it 'runs on designated pipeline' do - group = describe_successfully do - it('runs on nightly', only: { pipeline: :nightly }) {} - it('does not run in not_nightly', only: { pipeline: :not_nightly }) {} - it('runs on nightly given an array', only: { pipeline: [:canary, :nightly] }) {} - it('does not run in not_nightly given an array', only: { pipeline: [:not_nightly, :canary] }) {} - end - - aggregate_failures do - expect(group.examples[0].execution_result.status).to eq(:passed) - expect(group.examples[1].execution_result.status).to eq(:pending) - expect(group.examples[2].execution_result.status).to eq(:passed) - expect(group.examples[3].execution_result.status).to eq(:pending) - end - end - end - end - end end diff --git a/qa/spec/support/helpers/stub_env.rb b/qa/spec/support/helpers/stub_env.rb index 8ad864dbec8..de8d2f47adf 100644 --- a/qa/spec/support/helpers/stub_env.rb +++ b/qa/spec/support/helpers/stub_env.rb @@ -15,7 +15,7 @@ module Helpers private - STUBBED_KEY = '__STUBBED__'.freeze + STUBBED_KEY = '__STUBBED__' def add_stubbed_value(key, value) allow(ENV).to receive(:[]).with(key).and_return(value) |