diff options
Diffstat (limited to 'qa')
| -rw-r--r-- | qa/README.md | 25 | ||||
| -rw-r--r-- | qa/qa/scenario/shared_attributes.rb | 1 | ||||
| -rw-r--r-- | qa/qa/scenario/template.rb | 3 | ||||
| -rw-r--r-- | qa/qa/specs/features/browser_ui/5_package/.gitkeep | 0 | ||||
| -rw-r--r-- | qa/spec/scenario/template_spec.rb | 42 |
5 files changed, 65 insertions, 6 deletions
diff --git a/qa/README.md b/qa/README.md index 8f41327fb15..7ed4d63a589 100644 --- a/qa/README.md +++ b/qa/README.md @@ -178,11 +178,13 @@ another test has `:ldap` and `:quarantine` metadata. If the tests are run with `--tag smoke --tag quarantine`, only the first test will run. The test with `:ldap` will not run even though it also has `:quarantine`. -### Running tests with a feature flag enabled +### Running tests with a feature flag enabled or disabled -Tests can be run with with a feature flag enabled by using the command-line -option `--enable-feature FEATURE_FLAG`. For example, to enable the feature flag -that enforces Gitaly request limits, you would use the command: +Tests can be run with with a feature flag enabled or disabled by using the command-line +option `--enable-feature FEATURE_FLAG` or `--disable-feature FEATURE_FLAG`. + +For example, to enable the feature flag that enforces Gitaly request limits, +you would use the command: ``` bundle exec bin/qa Test::Instance::All http://localhost:3000 --enable-feature gitaly_enforce_requests_limits @@ -193,9 +195,20 @@ feature flag ([via the API](https://docs.gitlab.com/ee/api/features.html)), run all the tests in the `Test::Instance::All` scenario, and then disable the feature flag again. +Similarly, to disable the feature flag that enforces Gitaly request limits, +you would use the command: + +``` +bundle exec bin/qa Test::Instance::All http://localhost:3000 --disable-feature gitaly_enforce_requests_limits +``` +This will instruct the QA framework to disable the `gitaly_enforce_requests_limits` +feature flag ([via the API](https://docs.gitlab.com/ee/api/features.html)) if not already disabled, +run all the tests in the `Test::Instance::All` scenario, and then enable the +feature flag again if it was enabled earlier. + Note: the QA framework doesn't currently allow you to easily toggle a feature flag during a single test, [as you can in unit tests](https://docs.gitlab.com/ee/development/feature_flags.html#specs), but [that capability is planned](https://gitlab.com/gitlab-org/quality/team-tasks/issues/77). -Note also that the `--` separator isn't used because `--enable-feature` is a QA -framework option, not an `rspec` option. +Note also that the `--` separator isn't used because `--enable-feature` and `--disable-feature` +are QA framework options, not `rspec` options. diff --git a/qa/qa/scenario/shared_attributes.rb b/qa/qa/scenario/shared_attributes.rb index bb45c4ce4cb..e2eaca42277 100644 --- a/qa/qa/scenario/shared_attributes.rb +++ b/qa/qa/scenario/shared_attributes.rb @@ -7,6 +7,7 @@ module QA attribute :gitlab_address, '--address URL', 'Address of the instance to test' attribute :enable_feature, '--enable-feature FEATURE_FLAG', 'Enable a feature before running tests' + attribute :disable_feature, '--disable-feature FEATURE_FLAG', 'Disable a feature before running tests' attribute :parallel, '--parallel', 'Execute tests in parallel' attribute :loop, '--loop', 'Execute test repeatedly' end diff --git a/qa/qa/scenario/template.rb b/qa/qa/scenario/template.rb index 74d4c8f8757..0d517dffee8 100644 --- a/qa/qa/scenario/template.rb +++ b/qa/qa/scenario/template.rb @@ -30,6 +30,8 @@ module QA Runtime::Feature.enable(options[:enable_feature]) if options.key?(:enable_feature) + Runtime::Feature.disable(options[:disable_feature]) if options.key?(:disable_feature) && (@feature_enabled = Runtime::Feature.enabled?(options[:disable_feature])) + Specs::Runner.perform do |specs| specs.tty = true specs.tags = self.class.focus @@ -37,6 +39,7 @@ module QA end ensure Runtime::Feature.disable(options[:enable_feature]) if options.key?(:enable_feature) + Runtime::Feature.enable(options[:disable_feature]) if options.key?(:disable_feature) && @feature_enabled end def extract_option(name, options, args) diff --git a/qa/qa/specs/features/browser_ui/5_package/.gitkeep b/qa/qa/specs/features/browser_ui/5_package/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 --- a/qa/qa/specs/features/browser_ui/5_package/.gitkeep +++ /dev/null diff --git a/qa/spec/scenario/template_spec.rb b/qa/spec/scenario/template_spec.rb index f97fc22daf9..65793734548 100644 --- a/qa/spec/scenario/template_spec.rb +++ b/qa/spec/scenario/template_spec.rb @@ -17,6 +17,24 @@ describe QA::Scenario::Template do expect(feature).to have_received(:enable).with('a-feature') end + it 'allows a feature to be disabled' do + allow(QA::Runtime::Feature).to receive(:enabled?) + .with('another-feature').and_return(true) + + subject.perform({ disable_feature: 'another-feature' }) + + expect(feature).to have_received(:disable).with('another-feature') + end + + it 'does not disable a feature if already disabled' do + allow(QA::Runtime::Feature).to receive(:enabled?) + .with('another-feature').and_return(false) + + subject.perform({ disable_feature: 'another-feature' }) + + expect(feature).not_to have_received(:disable).with('another-feature') + end + it 'ensures an enabled feature is disabled afterwards' do allow(QA::Specs::Runner).to receive(:perform).and_raise('failed test') @@ -25,4 +43,28 @@ describe QA::Scenario::Template do expect(feature).to have_received(:enable).with('a-feature') expect(feature).to have_received(:disable).with('a-feature') end + + it 'ensures a disabled feature is enabled afterwards' do + allow(QA::Specs::Runner).to receive(:perform).and_raise('failed test') + + allow(QA::Runtime::Feature).to receive(:enabled?) + .with('another-feature').and_return(true) + + expect { subject.perform({ disable_feature: 'another-feature' }) }.to raise_error('failed test') + + expect(feature).to have_received(:disable).with('another-feature') + expect(feature).to have_received(:enable).with('another-feature') + end + + it 'ensures a disabled feature is not enabled afterwards if it was disabled earlier' do + allow(QA::Specs::Runner).to receive(:perform).and_raise('failed test') + + allow(QA::Runtime::Feature).to receive(:enabled?) + .with('another-feature').and_return(false) + + expect { subject.perform({ disable_feature: 'another-feature' }) }.to raise_error('failed test') + + expect(feature).not_to have_received(:disable).with('another-feature') + expect(feature).not_to have_received(:enable).with('another-feature') + end end |