diff options
author | João Cunha <j.a.cunha@gmail.com> | 2019-02-26 10:33:32 +0000 |
---|---|---|
committer | João Cunha <j.a.cunha@gmail.com> | 2019-02-26 10:33:32 +0000 |
commit | ed7065bbdb6f6fad3cc8fb369267fa811767c8a5 (patch) | |
tree | b3fda377c51504a6d21da1742cac3de7fd668ef0 /qa/spec/spec_helper.rb | |
parent | 26c7d04cd06d1126e4a4a7a1c9308d12de77172e (diff) | |
parent | 094d740886eaf62fd219dacd11aa9a966758a962 (diff) | |
download | gitlab-ce-56937-edit-knative-domain-after-it-has-been-deployed.tar.gz |
Merge branch 'master' into 56937-edit-knative-domain-after-it-has-been-deployed56937-edit-knative-domain-after-it-has-been-deployed
Diffstat (limited to 'qa/spec/spec_helper.rb')
-rw-r--r-- | qa/spec/spec_helper.rb | 91 |
1 files changed, 52 insertions, 39 deletions
diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb index 0f3cf5f4408..cbdd6e881b1 100644 --- a/qa/spec/spec_helper.rb +++ b/qa/spec/spec_helper.rb @@ -1,42 +1,21 @@ require_relative '../qa' +require 'rspec/retry' -Dir[::File.join(__dir__, 'support', '**', '*.rb')].each { |f| require f } +%w[helpers shared_examples].each do |d| + Dir[::File.join(__dir__, d, '**', '*.rb')].each { |f| require f } +end RSpec.configure do |config| - ServerNotRespondingError = Class.new(RuntimeError) - - # The login page could take some time to load the first time it is visited. - # We visit the login page and wait for it to properly load only once at the beginning of the suite. - config.before(:suite) do - if QA::Runtime::Scenario.respond_to?(:gitlab_address) - QA::Runtime::Browser.visit(:gitlab, QA::Page::Main::Login) - - unless QA::Page::Main::Login.perform(&:page_loaded?) - raise ServerNotRespondingError, "Login page did not load at #{QA::Page::Main::Login.perform(&:current_url)}" - end + config.before(:context) do + if self.class.metadata.keys.include?(:quarantine) + skip_or_run_quarantined_tests(self.class.metadata.keys, config.inclusion_filter.rules.keys) end end config.before do |example| QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug? - # If quarantine is tagged, skip tests that have other metadata unless - # they're also tagged. This lets us run quarantined tests in a particular - # category without running tests in other categories. - # E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged - # 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests - # using `--tag quarantine --tag smoke`, without this check we'd end up - # running that ldap test as well. - if config.inclusion_filter[:quarantine] - skip("Running tests tagged with all of #{config.inclusion_filter.rules.keys}") unless quarantine_and_optional_other_tag?(example, config) - end - end - - config.before(:each, :quarantine) do |example| - # Skip tests in quarantine unless we explicitly focus on them - # We could use an exclusion filter, but this way the test report will list - # the quarantined tests when they're not run so that we're aware of them - skip('In quarantine') unless config.inclusion_filter[:quarantine] + skip_or_run_quarantined_tests(example.metadata.keys, config.inclusion_filter.rules.keys) end config.expect_with :rspec do |expectations| @@ -53,20 +32,54 @@ RSpec.configure do |config| config.profile_examples = 10 config.order = :random Kernel.srand config.seed + + # show retry status in spec process + config.verbose_retry = true + + # show exception that triggers a retry if verbose_retry is set to true + config.display_try_failure_messages = true + + config.around do |example| + retry_times = example.metadata.keys.include?(:quarantine) ? 1 : 3 + example.run_with_retry retry: retry_times + end +end + +# Skip tests in quarantine unless we explicitly focus on them. +# Skip the entire context if a context is tagged. This avoids running before +# blocks unnecessarily. +# If quarantine is focussed, skip tests/contexts that have other metadata +# unless they're also focussed. This lets us run quarantined tests in a +# particular category without running tests in other categories. +# E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged +# 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests +# using `--tag quarantine --tag smoke`, without this check we'd end up +# running that ldap test as well. +# We could use an exclusion filter, but this way the test report will list +# the quarantined tests when they're not run so that we're aware of them +def skip_or_run_quarantined_tests(metadata_keys, filter_keys) + included_filters = filters_other_than_quarantine(filter_keys) + + if filter_keys.include?(:quarantine) + skip("Only running tests tagged with :quarantine and any of #{included_filters}") unless quarantine_and_optional_other_tag?(metadata_keys, included_filters) + else + skip('In quarantine') if metadata_keys.include?(:quarantine) + end +end + +def filters_other_than_quarantine(filter_keys) + filter_keys.reject { |key| key == :quarantine } end # Checks if a test has the 'quarantine' tag and other tags in the inclusion filter. # # Returns true if -# - the example metadata includes the quarantine tag -# - and the metadata and inclusion filter both have any other tag -# - or no other tags are in the inclusion filter -def quarantine_and_optional_other_tag?(example, config) - return false unless example.metadata.keys.include? :quarantine - - filters_other_than_quarantine = config.inclusion_filter.rules.keys.reject { |key| key == :quarantine } - - return true if filters_other_than_quarantine.empty? +# - the metadata includes the quarantine tag +# - and the metadata and inclusion filter both have any other tag +# - or no other tags are in the inclusion filter +def quarantine_and_optional_other_tag?(metadata_keys, included_filters) + return false unless metadata_keys.include? :quarantine + return true if included_filters.empty? - filters_other_than_quarantine.any? { |key| example.metadata.keys.include? key } + included_filters.any? { |key| metadata_keys.include? key } end |