diff options
author | Igor Drozdov <idrozdov@gitlab.com> | 2019-04-01 17:36:11 +0300 |
---|---|---|
committer | Igor Drozdov <idrozdov@gitlab.com> | 2019-04-01 17:36:11 +0300 |
commit | 04bb35a4b562fd57b14c55645bb1848a50cdef56 (patch) | |
tree | 1bd1ac2af6a5c088ac2529cdbccceeca402d3ebe /qa | |
parent | ade207e575ab846f6d354aaccc1382a6e512dd0d (diff) | |
parent | b8118a65d595040bfce2d83d5e38dd63ebfedb58 (diff) | |
download | gitlab-ce-id-split-self-approval-restrictions.tar.gz |
Merge branch 'master' into id-split-self-approval-restrictionsid-split-self-approval-restrictions
Diffstat (limited to 'qa')
88 files changed, 461 insertions, 183 deletions
diff --git a/qa/.gitignore b/qa/.gitignore index 102f7e5e54d..b0ae074ac07 100644 --- a/qa/.gitignore +++ b/qa/.gitignore @@ -1,3 +1,3 @@ tmp/ .ruby-version -urls.txt +urls.yml diff --git a/qa/README.md b/qa/README.md index 735868e7640..7d66f7d5abc 100644 --- a/qa/README.md +++ b/qa/README.md @@ -55,16 +55,19 @@ You can also supply specific tests to run as another parameter. For example, to run the repository-related specs, you can execute: ``` -bin/qa Test::Instance::All http://localhost qa/specs/features/repository/ +bin/qa Test::Instance::All http://localhost -- qa/specs/features/browser_ui/3_create/repository ``` Since the arguments would be passed to `rspec`, you could use all `rspec` options there. For example, passing `--backtrace` and also line number: ``` -bin/qa Test::Instance::All http://localhost qa/specs/features/project/create_spec.rb:3 --backtrace +bin/qa Test::Instance::All http://localhost -- qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb:6 --backtrace ``` +Note that the separator `--` is required; all subsequent options will be +ignored by the QA framework and passed to `rspec`. + ### Overriding the authenticated user Unless told otherwise, the QA tests will run as the default `root` user seeded @@ -117,7 +120,7 @@ tests that are expected to fail while a fix is in progress (similar to how can be used). ``` -bin/qa Test::Instance::All http://localhost --tag quarantine +bin/qa Test::Instance::All http://localhost -- --tag quarantine ``` If `quarantine` is used with other tags, tests will only be run if they have at @@ -128,3 +131,25 @@ For example, suppose one test has `:smoke` and `:quarantine` metadata, and another test has `:ldap` and `:quarantine` metadata. If the tests are run with `--tag smoke --tag quarantine`, only the first test will run. The test with `:ldap` will not run even though it also has `:quarantine`. + +### Running tests with a feature flag enabled + +Tests can be run with with a feature flag enabled by using the command-line +option `--enable-feature FEATURE_FLAG`. For example, to enable the feature flag +that enforces Gitaly request limits, you would use the command: + +``` +bin/qa Test::Instance::All http://localhost --enable-feature gitaly_enforce_requests_limits +``` + +This will instruct the QA framework to enable the `gitaly_enforce_requests_limits` +feature flag ([via the API](https://docs.gitlab.com/ee/api/features.html)), run +all the tests in the `Test::Instance::All` scenario, and then disable the +feature flag again. + +Note: the QA framework doesn't currently allow you to easily toggle a feature +flag during a single test, [as you can in unit tests](https://docs.gitlab.com/ee/development/feature_flags.html#specs), +but [that capability is planned](https://gitlab.com/gitlab-org/quality/team-tasks/issues/77). + +Note also that the `--` separator isn't used because `--enable-feature` is a QA +framework option, not an `rspec` option.
\ No newline at end of file diff --git a/qa/Rakefile b/qa/Rakefile index b6ad09f9b00..7ac018f7286 100644 --- a/qa/Rakefile +++ b/qa/Rakefile @@ -16,3 +16,25 @@ desc "Generate Performance Testdata" task :generate_perf_testdata do QA::Tools::GeneratePerfTestdata.new.run end + +desc "Run artillery load tests" +task :run_artillery_load_tests do + unless ENV['HOST_URL'] && ENV['LARGE_ISSUE_URL'] && ENV['LARGE_MR_URL'] + urls_file = ENV['URLS_FILE_PATH'] || 'urls.yml' + + unless File.exist?(urls_file) + raise "\n#{urls_file} file is missing. Please provide correct URLS_FILE_PATH or all of HOST_URL, LARGE_ISSUE_URL and LARGE_MR_URL\n\n" + end + + urls = YAML.safe_load(File.read(urls_file)) + ENV['HOST_URL'] = urls["host"] + ENV['LARGE_ISSUE_URL'] = urls["large_issue"] + ENV['LARGE_MR_URL'] = urls["large_mr"] + end + + sh('artillery run load/artillery.yml -o report.json') + sh('artillery report report.json -o report.html && rm report.json') +end + +desc "Generate data and run load tests" +task generate_data_and_run_load_test: [:generate_perf_testdata, :run_artillery_load_tests] diff --git a/qa/load/artillery.yml b/qa/load/artillery.yml new file mode 100644 index 00000000000..17d253ec480 --- /dev/null +++ b/qa/load/artillery.yml @@ -0,0 +1,25 @@ +config: + target: "{{ $processEnvironment.HOST_URL }}" + http: + pool: 10 # All HTTP requests from all virtual users will be sent over the same <pool> connections. + # This also means that there is a limit on the number of requests sent per second. + phases: + - duration: 30 + arrivalRate: 10 + name: "Warm up" + - duration: 90 + arrivalRate: 10 + rampTo: 100 + name: "Gradual ramp up" + - duration: 90 + arrivalRate: 100 + name: "Sustained max load" +scenarios: + - name: "Visit large issue url" + flow: + - get: + url: "{{ $processEnvironment.LARGE_ISSUE_URL }}" + - name: "Visit large MR url" + flow: + - get: + url: "{{ $processEnvironment.LARGE_MR_URL }}" @@ -17,6 +17,7 @@ module QA autoload :Env, 'qa/runtime/env' autoload :Address, 'qa/runtime/address' autoload :Path, 'qa/runtime/path' + autoload :Feature, 'qa/runtime/feature' autoload :Fixtures, 'qa/runtime/fixtures' autoload :Logger, 'qa/runtime/logger' @@ -89,6 +90,7 @@ module QA autoload :Bootable, 'qa/scenario/bootable' autoload :Actable, 'qa/scenario/actable' autoload :Template, 'qa/scenario/template' + autoload :SharedAttributes, 'qa/scenario/shared_attributes' ## # Test scenario entrypoints. diff --git a/qa/qa/page/dashboard/groups.rb b/qa/qa/page/dashboard/groups.rb index 7a07515de62..52853376f17 100644 --- a/qa/qa/page/dashboard/groups.rb +++ b/qa/qa/page/dashboard/groups.rb @@ -19,11 +19,13 @@ module QA has_filtered_group?(name) end - def go_to_group(name) + def click_group(name) + raise "Group with name #{name} not found!" unless has_group?(name) + click_link name end - def go_to_new_group + def click_new_group click_on 'New group' end end diff --git a/qa/qa/page/dashboard/snippet/new.rb b/qa/qa/page/dashboard/snippet/new.rb index a637b869d2f..4a8e65e20af 100644 --- a/qa/qa/page/dashboard/snippet/new.rb +++ b/qa/qa/page/dashboard/snippet/new.rb @@ -37,7 +37,7 @@ module QA text_area.set content end - def create_snippet + def click_create_snippet_button click_element :create_snippet_button end diff --git a/qa/qa/page/group/show.rb b/qa/qa/page/group/show.rb index 41716326685..d4c4be0d6ca 100644 --- a/qa/qa/page/group/show.rb +++ b/qa/qa/page/group/show.rb @@ -18,7 +18,7 @@ module QA element :no_result_text, 'No groups or projects matched your search' # rubocop:disable QA/ElementWithPattern end - def go_to_subgroup(name) + def click_subgroup(name) click_link name end diff --git a/qa/qa/page/label/index.rb b/qa/qa/page/label/index.rb index de0cfa9f293..e73d40b37ac 100644 --- a/qa/qa/page/label/index.rb +++ b/qa/qa/page/label/index.rb @@ -18,7 +18,7 @@ module QA element :label_svg end - def go_to_new_label + def click_new_label_button # The 'labels.svg' takes a fraction of a second to load after which the "New label" button shifts up a bit # This can cause webdriver to miss the hit so we wait for the svg to load (implicitly with has_element?) # before clicking the button. diff --git a/qa/qa/page/label/new.rb b/qa/qa/page/label/new.rb index b5422dc9400..9c8cf5f07e6 100644 --- a/qa/qa/page/label/new.rb +++ b/qa/qa/page/label/new.rb @@ -9,7 +9,7 @@ module QA element :label_create_button end - def create_label + def click_label_create_button click_element :label_create_button end diff --git a/qa/qa/page/main/menu.rb b/qa/qa/page/main/menu.rb index 1b3445b0064..e98d531c86e 100644 --- a/qa/qa/page/main/menu.rb +++ b/qa/qa/page/main/menu.rb @@ -47,7 +47,7 @@ module QA end end - def go_to_admin_area + def click_admin_area within_top_menu { click_element :admin_area_link } end @@ -57,7 +57,7 @@ module QA end end - def go_to_profile_settings + def click_settings_link retry_until(reload: false) do within_user_menu do click_link 'Settings' @@ -67,7 +67,7 @@ module QA end end - def go_to_snippets + def click_snippets_link click_element :snippets_link end diff --git a/qa/qa/page/merge_request/show.rb b/qa/qa/page/merge_request/show.rb index c0411db6505..5aef868a805 100644 --- a/qa/qa/page/merge_request/show.rb +++ b/qa/qa/page/merge_request/show.rb @@ -145,11 +145,11 @@ module QA click_element :squash_checkbox end - def go_to_discussions_tab + def click_discussions_tab click_element :notes_tab end - def go_to_diffs_tab + def click_diffs_tab click_element :diffs_tab end diff --git a/qa/qa/page/profile/personal_access_tokens.rb b/qa/qa/page/profile/personal_access_tokens.rb index 8c12eff5cf1..c9a5a1b3ca0 100644 --- a/qa/qa/page/profile/personal_access_tokens.rb +++ b/qa/qa/page/profile/personal_access_tokens.rb @@ -26,7 +26,7 @@ module QA check_element(:api_radio) end - def create_token + def click_create_token_button click_element(:create_token_button) end diff --git a/qa/qa/page/project/activity.rb b/qa/qa/page/project/activity.rb index 56fbaa90790..afd4f49a844 100644 --- a/qa/qa/page/project/activity.rb +++ b/qa/qa/page/project/activity.rb @@ -6,7 +6,7 @@ module QA element :push_events, "event_filter_link EventFilter::PUSH, _('Push events')" # rubocop:disable QA/ElementWithPattern end - def go_to_push_events + def click_push_events click_on 'Push events' end end diff --git a/qa/qa/page/project/issue/index.rb b/qa/qa/page/project/issue/index.rb index 1035bf74a43..e987d279e21 100644 --- a/qa/qa/page/project/issue/index.rb +++ b/qa/qa/page/project/issue/index.rb @@ -7,7 +7,7 @@ module QA element :issue_link, 'link_to issue.title' # rubocop:disable QA/ElementWithPattern end - def go_to_issue(title) + def click_issue_link(title) click_link(title) end end diff --git a/qa/qa/page/project/job/show.rb b/qa/qa/page/project/job/show.rb index 9c218f4ed8b..5853f487f0b 100644 --- a/qa/qa/page/project/job/show.rb +++ b/qa/qa/page/project/job/show.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module QA::Page module Project::Job class Show < QA::Page::Base @@ -31,7 +33,9 @@ module QA::Page private def loaded?(wait: 60) - has_element?(:build_trace, wait: wait) + wait(reload: true, max: wait, interval: 1) do + has_element?(:build_trace, wait: 1) + end end def completed?(timeout: 60) diff --git a/qa/qa/page/project/menu.rb b/qa/qa/page/project/menu.rb index 46dfe87fe25..3fe048f752a 100644 --- a/qa/qa/page/project/menu.rb +++ b/qa/qa/page/project/menu.rb @@ -30,7 +30,7 @@ module QA end end - def go_to_activity + def click_activity within_sidebar do click_element(:activity_link) end diff --git a/qa/qa/page/project/milestone/new.rb b/qa/qa/page/project/milestone/new.rb index 992ef89004b..0db8d2cf766 100644 --- a/qa/qa/page/project/milestone/new.rb +++ b/qa/qa/page/project/milestone/new.rb @@ -17,7 +17,7 @@ module QA fill_element :milestone_description, description end - def create_new_milestone + def click_milestone_create_button click_element :milestone_create_button end end diff --git a/qa/qa/page/project/new.rb b/qa/qa/page/project/new.rb index 552b2293115..f841afbbffd 100644 --- a/qa/qa/page/project/new.rb +++ b/qa/qa/page/project/new.rb @@ -35,7 +35,7 @@ module QA end end - def go_to_import_project + def click_import_project click_on 'Import project' end @@ -51,7 +51,7 @@ module QA click_on 'Create project' end - def go_to_create_from_template + def click_create_from_template_tab click_element(:project_create_from_template_tab) end @@ -59,7 +59,7 @@ module QA choose visibility end - def go_to_github_import + def click_github_link click_link 'GitHub' end end diff --git a/qa/qa/page/project/operations/environments/index.rb b/qa/qa/page/project/operations/environments/index.rb index 63965a57edd..610a34385b1 100644 --- a/qa/qa/page/project/operations/environments/index.rb +++ b/qa/qa/page/project/operations/environments/index.rb @@ -10,7 +10,7 @@ module QA element :environment_link end - def go_to_environment(environment_name) + def click_environment_link(environment_name) wait(reload: false) do find(element_selector_css(:environment_link), text: environment_name).click end diff --git a/qa/qa/page/project/pipeline/index.rb b/qa/qa/page/project/pipeline/index.rb index 19d83ecc4f4..0373a12f597 100644 --- a/qa/qa/page/project/pipeline/index.rb +++ b/qa/qa/page/project/pipeline/index.rb @@ -5,7 +5,7 @@ module QA::Page element :pipeline_link, 'class="js-pipeline-url-link"' # rubocop:disable QA/ElementWithPattern end - def go_to_latest_pipeline + def click_on_latest_pipeline css = '.js-pipeline-url-link' link = wait(reload: false) do diff --git a/qa/qa/page/project/pipeline/show.rb b/qa/qa/page/project/pipeline/show.rb index 6f8a66bf527..9c0b55b1c4c 100644 --- a/qa/qa/page/project/pipeline/show.rb +++ b/qa/qa/page/project/pipeline/show.rb @@ -42,11 +42,11 @@ module QA::Page end end - def go_to_job(job_name) + def click_job(job_name) find_element(:job_link, text: job_name).click end - def go_to_first_job + def click_on_first_job css = '.js-pipeline-graph-job-link' wait(reload: false) do diff --git a/qa/qa/page/project/settings/protected_branches.rb b/qa/qa/page/project/settings/protected_branches.rb index 76591a4e3fe..399a49d2420 100644 --- a/qa/qa/page/project/settings/protected_branches.rb +++ b/qa/qa/page/project/settings/protected_branches.rb @@ -32,22 +32,22 @@ module QA end def allow_no_one_to_push - click_allow(:push, 'No one') + go_to_allow(:push, 'No one') end def allow_devs_and_maintainers_to_push - click_allow(:push, 'Developers + Maintainers') + go_to_allow(:push, 'Developers + Maintainers') end # @deprecated alias_method :allow_devs_and_masters_to_push, :allow_devs_and_maintainers_to_push def allow_no_one_to_merge - click_allow(:merge, 'No one') + go_to_allow(:merge, 'No one') end def allow_devs_and_maintainers_to_merge - click_allow(:merge, 'Developers + Maintainers') + go_to_allow(:merge, 'Developers + Maintainers') end # @deprecated @@ -59,7 +59,7 @@ module QA private - def click_allow(action, text) + def go_to_allow(action, text) click_element :"allowed_to_#{action}_select" within_element(:"allowed_to_#{action}_dropdown") do diff --git a/qa/qa/page/project/show.rb b/qa/qa/page/project/show.rb index 9c21d9ddbfa..1a9a2fd413f 100644 --- a/qa/qa/page/project/show.rb +++ b/qa/qa/page/project/show.rb @@ -76,13 +76,13 @@ module QA click_on 'Fork' end - def go_to_file(filename) + def click_file(filename) within_element(:file_tree) do click_on filename end end - def go_to_commit(commit_msg) + def click_commit(commit_msg) within_element(:file_tree) do click_on commit_msg end diff --git a/qa/qa/page/project/sub_menus/operations.rb b/qa/qa/page/project/sub_menus/operations.rb index cf9fc453565..24a99a9464c 100644 --- a/qa/qa/page/project/sub_menus/operations.rb +++ b/qa/qa/page/project/sub_menus/operations.rb @@ -14,7 +14,7 @@ module QA end end - def click_operations_environments + def go_to_operations_environments hover_operations do within_submenu do click_element(:operations_environments_link) @@ -22,7 +22,7 @@ module QA end end - def click_operations_kubernetes + def go_to_operations_kubernetes hover_operations do within_submenu do click_link('Kubernetes') diff --git a/qa/qa/page/project/sub_menus/repository.rb b/qa/qa/page/project/sub_menus/repository.rb index 29eaa9a74de..4cc73a6b25a 100644 --- a/qa/qa/page/project/sub_menus/repository.rb +++ b/qa/qa/page/project/sub_menus/repository.rb @@ -20,7 +20,7 @@ module QA end end - def click_repository_branches + def go_to_repository_branches hover_repository do within_submenu do click_element(:branches_link) diff --git a/qa/qa/page/project/sub_menus/settings.rb b/qa/qa/page/project/sub_menus/settings.rb index 62c594c0210..22743ebd0a1 100644 --- a/qa/qa/page/project/sub_menus/settings.rb +++ b/qa/qa/page/project/sub_menus/settings.rb @@ -14,7 +14,7 @@ module QA end end - def click_ci_cd_settings + def go_to_ci_cd_settings hover_settings do within_submenu do click_link('CI / CD') @@ -22,7 +22,7 @@ module QA end end - def click_members_settings + def go_to_members_settings hover_settings do within_submenu do click_element :link_members_settings @@ -30,7 +30,7 @@ module QA end end - def click_repository_settings + def go_to_repository_settings hover_settings do within_submenu do click_link('Repository') @@ -38,7 +38,7 @@ module QA end end - def go_to_settings + def click_settings within_sidebar do click_on 'Settings' end diff --git a/qa/qa/page/project/wiki/edit.rb b/qa/qa/page/project/wiki/edit.rb index 8d0eafa1818..1ccd67349c3 100644 --- a/qa/qa/page/project/wiki/edit.rb +++ b/qa/qa/page/project/wiki/edit.rb @@ -9,15 +9,7 @@ module QA element :edit_page_link, 'Edit' # rubocop:disable QA/ElementWithPattern end - def go_to_new_page - click_on 'New page' - end - - def got_to_view_history_page - click_on 'Page history' - end - - def go_to_edit_page + def click_edit click_on 'Edit' end end diff --git a/qa/qa/page/project/wiki/new.rb b/qa/qa/page/project/wiki/new.rb index b90e03be36a..792eba4bab7 100644 --- a/qa/qa/page/project/wiki/new.rb +++ b/qa/qa/page/project/wiki/new.rb @@ -23,7 +23,7 @@ module QA element :svg_content end - def go_to_create_first_page + def click_create_your_first_page_button # The svg takes a fraction of a second to load after which the # "Create your first page" button shifts up a bit. This can cause # webdriver to miss the hit so we wait for the svg to load before diff --git a/qa/qa/page/project/wiki/show.rb b/qa/qa/page/project/wiki/show.rb index dffbc5d60a2..f79ad510084 100644 --- a/qa/qa/page/project/wiki/show.rb +++ b/qa/qa/page/project/wiki/show.rb @@ -11,7 +11,7 @@ module QA element :clone_repository_link, 'Clone repository' # rubocop:disable QA/ElementWithPattern end - def go_to_clone_repository + def click_clone_repository click_on 'Clone repository' end end diff --git a/qa/qa/resource/api_fabricator.rb b/qa/qa/resource/api_fabricator.rb index 98eebac0880..de04467ff5b 100644 --- a/qa/qa/resource/api_fabricator.rb +++ b/qa/qa/resource/api_fabricator.rb @@ -8,9 +8,6 @@ module QA module ApiFabricator include Capybara::DSL - HTTP_STATUS_OK = 200 - HTTP_STATUS_CREATED = 201 - ResourceNotFoundError = Class.new(RuntimeError) ResourceFabricationFailedError = Class.new(RuntimeError) ResourceURLMissingError = Class.new(RuntimeError) diff --git a/qa/qa/resource/branch.rb b/qa/qa/resource/branch.rb index bd52c4abe02..a45dd030625 100644 --- a/qa/qa/resource/branch.rb +++ b/qa/qa/resource/branch.rb @@ -46,7 +46,7 @@ module QA # to `allow_to_push` variable. return branch unless @protected - Page::Project::Menu.perform(&:click_repository_settings) + Page::Project::Menu.perform(&:go_to_repository_settings) Page::Project::Settings::Repository.perform do |setting| setting.expand_protected_branches do |page| diff --git a/qa/qa/resource/ci_variable.rb b/qa/qa/resource/ci_variable.rb index 0570c47d41c..d82de4cb816 100644 --- a/qa/qa/resource/ci_variable.rb +++ b/qa/qa/resource/ci_variable.rb @@ -15,7 +15,7 @@ module QA def fabricate! project.visit! - Page::Project::Menu.perform(&:click_ci_cd_settings) + Page::Project::Menu.perform(&:go_to_ci_cd_settings) Page::Project::Settings::CICD.perform do |setting| setting.expand_ci_variables do |page| diff --git a/qa/qa/resource/deploy_key.rb b/qa/qa/resource/deploy_key.rb index 9565598efb0..869e2a71e47 100644 --- a/qa/qa/resource/deploy_key.rb +++ b/qa/qa/resource/deploy_key.rb @@ -23,7 +23,7 @@ module QA def fabricate! project.visit! - Page::Project::Menu.perform(&:click_repository_settings) + Page::Project::Menu.perform(&:go_to_repository_settings) Page::Project::Settings::Repository.perform do |setting| setting.expand_deploy_keys do |page| diff --git a/qa/qa/resource/deploy_token.rb b/qa/qa/resource/deploy_token.rb index cee4422f6b4..fca5ed83c87 100644 --- a/qa/qa/resource/deploy_token.rb +++ b/qa/qa/resource/deploy_token.rb @@ -32,7 +32,7 @@ module QA project.visit! Page::Project::Menu.act do - click_repository_settings + go_to_repository_settings end Page::Project::Settings::Repository.perform do |setting| diff --git a/qa/qa/resource/group.rb b/qa/qa/resource/group.rb index d7f9ec6a836..0b567a474c8 100644 --- a/qa/qa/resource/group.rb +++ b/qa/qa/resource/group.rb @@ -21,7 +21,7 @@ module QA Page::Group::Show.perform do |group_show| if group_show.has_subgroup?(path) - group_show.go_to_subgroup(path) + group_show.click_subgroup(path) else group_show.go_to_new_subgroup diff --git a/qa/qa/resource/kubernetes_cluster.rb b/qa/qa/resource/kubernetes_cluster.rb index 93a06be6818..27ab7b60211 100644 --- a/qa/qa/resource/kubernetes_cluster.rb +++ b/qa/qa/resource/kubernetes_cluster.rb @@ -16,7 +16,7 @@ module QA @project.visit! Page::Project::Menu.perform( - &:click_operations_kubernetes) + &:go_to_operations_kubernetes) Page::Project::Operations::Kubernetes::Index.perform( &:add_kubernetes_cluster) diff --git a/qa/qa/resource/label.rb b/qa/qa/resource/label.rb index c0869cb1f2a..7c899db31f3 100644 --- a/qa/qa/resource/label.rb +++ b/qa/qa/resource/label.rb @@ -25,13 +25,13 @@ module QA project.visit! Page::Project::Menu.perform(&:go_to_labels) - Page::Label::Index.perform(&:go_to_new_label) + Page::Label::Index.perform(&:click_new_label_button) Page::Label::New.perform do |page| page.fill_title(@title) page.fill_description(@description) page.fill_color(@color) - page.create_label + page.click_label_create_button end end end diff --git a/qa/qa/resource/personal_access_token.rb b/qa/qa/resource/personal_access_token.rb index b8dd0a3562f..f5c632cd8d2 100644 --- a/qa/qa/resource/personal_access_token.rb +++ b/qa/qa/resource/personal_access_token.rb @@ -13,13 +13,13 @@ module QA end def fabricate! - Page::Main::Menu.perform(&:go_to_profile_settings) + Page::Main::Menu.perform(&:click_settings_link) Page::Profile::Menu.perform(&:click_access_tokens) Page::Profile::PersonalAccessTokens.perform do |page| page.fill_token_name(name || 'api-test-token') page.check_api - page.create_token + page.click_create_token_button end end end diff --git a/qa/qa/resource/project_imported_from_github.rb b/qa/qa/resource/project_imported_from_github.rb index 0d25e7dd842..a160cdb3273 100644 --- a/qa/qa/resource/project_imported_from_github.rb +++ b/qa/qa/resource/project_imported_from_github.rb @@ -18,11 +18,11 @@ module QA Page::Group::Show.perform(&:go_to_new_project) Page::Project::New.perform do |page| - page.go_to_import_project + page.click_import_project end Page::Project::New.perform do |page| - page.go_to_github_import + page.click_github_link end Page::Project::Import::Github.perform do |page| diff --git a/qa/qa/resource/project_milestone.rb b/qa/qa/resource/project_milestone.rb index a4d6657caff..8ace75f695a 100644 --- a/qa/qa/resource/project_milestone.rb +++ b/qa/qa/resource/project_milestone.rb @@ -28,7 +28,7 @@ module QA Page::Project::Milestone::New.perform do |milestone_new| milestone_new.set_title(@title) milestone_new.set_description(@description) - milestone_new.create_new_milestone + milestone_new.click_milestone_create_button end end end diff --git a/qa/qa/resource/repository/wiki_push.rb b/qa/qa/resource/repository/wiki_push.rb index 77c4c8a514d..95712300854 100644 --- a/qa/qa/resource/repository/wiki_push.rb +++ b/qa/qa/resource/repository/wiki_push.rb @@ -24,7 +24,7 @@ module QA @repository_http_uri ||= begin wiki.visit! Page::Project::Wiki::Show.act do - go_to_clone_repository + click_clone_repository choose_repository_clone_http repository_location.uri end diff --git a/qa/qa/resource/runner.rb b/qa/qa/resource/runner.rb index 08ae3f22117..3344ad3360a 100644 --- a/qa/qa/resource/runner.rb +++ b/qa/qa/resource/runner.rb @@ -29,7 +29,7 @@ module QA def fabricate! project.visit! - Page::Project::Menu.perform(&:click_ci_cd_settings) + Page::Project::Menu.perform(&:go_to_ci_cd_settings) Service::Runner.new(name).tap do |runner| Page::Project::Settings::CICD.perform do |settings| diff --git a/qa/qa/resource/sandbox.rb b/qa/qa/resource/sandbox.rb index 41ce857a8b8..942eea5cc40 100644 --- a/qa/qa/resource/sandbox.rb +++ b/qa/qa/resource/sandbox.rb @@ -20,9 +20,9 @@ module QA Page::Dashboard::Groups.perform do |page| if page.has_group?(path) - page.go_to_group(path) + page.click_group(path) else - page.go_to_new_group + page.click_new_group Page::Group::New.perform do |group| group.set_path(path) diff --git a/qa/qa/resource/settings/hashed_storage.rb b/qa/qa/resource/settings/hashed_storage.rb index 40c06768ffe..08bb95cfd4b 100644 --- a/qa/qa/resource/settings/hashed_storage.rb +++ b/qa/qa/resource/settings/hashed_storage.rb @@ -8,7 +8,7 @@ module QA raise ArgumentError unless traits.include?(:enabled) Page::Main::Login.perform(&:sign_in_using_credentials) - Page::Main::Menu.perform(&:go_to_admin_area) + Page::Main::Menu.perform(&:click_admin_area) Page::Admin::Menu.perform(&:go_to_repository_settings) Page::Admin::Settings::Repository.perform do |setting| diff --git a/qa/qa/resource/snippet.rb b/qa/qa/resource/snippet.rb index 1478f197570..f58d7b5113f 100644 --- a/qa/qa/resource/snippet.rb +++ b/qa/qa/resource/snippet.rb @@ -22,7 +22,7 @@ module QA page.set_visibility(@visibility) page.fill_file_name(@file_name) page.fill_file_content(@file_content) - page.create_snippet + page.click_create_snippet_button end end end diff --git a/qa/qa/resource/ssh_key.rb b/qa/qa/resource/ssh_key.rb index c6c97c8532f..4e399ae730e 100644 --- a/qa/qa/resource/ssh_key.rb +++ b/qa/qa/resource/ssh_key.rb @@ -14,7 +14,7 @@ module QA end def fabricate! - Page::Main::Menu.perform(&:go_to_profile_settings) + Page::Main::Menu.perform(&:click_settings_link) Page::Profile::Menu.perform(&:click_ssh_keys) Page::Profile::SSHKeys.perform do |page| diff --git a/qa/qa/resource/wiki.rb b/qa/qa/resource/wiki.rb index e942e9718a0..0a776f22e25 100644 --- a/qa/qa/resource/wiki.rb +++ b/qa/qa/resource/wiki.rb @@ -18,7 +18,7 @@ module QA Page::Project::Menu.perform { |menu_side| menu_side.click_wiki } Page::Project::Wiki::New.perform do |wiki_new| - wiki_new.go_to_create_first_page + wiki_new.click_create_your_first_page_button wiki_new.set_title(@title) wiki_new.set_content(@content) wiki_new.set_message(@message) diff --git a/qa/qa/runtime/address.rb b/qa/qa/runtime/address.rb index ffad3974b02..af0537dc17c 100644 --- a/qa/qa/runtime/address.rb +++ b/qa/qa/runtime/address.rb @@ -15,6 +15,13 @@ module QA @instance.to_s end end + + def self.valid?(value) + uri = URI.parse(value) + uri.is_a?(URI::HTTP) && !uri.host.nil? + rescue URI::InvalidURIError + false + end end end end diff --git a/qa/qa/runtime/api/client.rb b/qa/qa/runtime/api/client.rb index aff84c89f0e..d3327b49339 100644 --- a/qa/qa/runtime/api/client.rb +++ b/qa/qa/runtime/api/client.rb @@ -14,7 +14,7 @@ module QA def personal_access_token @personal_access_token ||= begin - # you can set the environment variable PERSONAL_ACCESS_TOKEN + # you can set the environment variable GITLAB_QA_ACCESS_TOKEN # to use a specific access token rather than create one from the UI Runtime::Env.personal_access_token ||= create_personal_access_token end diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb index dd0ddbdbd6b..03cae3c1fe6 100644 --- a/qa/qa/runtime/env.rb +++ b/qa/qa/runtime/env.rb @@ -53,7 +53,7 @@ module QA # specifies token that can be used for the api def personal_access_token - @personal_access_token ||= ENV['PERSONAL_ACCESS_TOKEN'] + @personal_access_token ||= ENV['GITLAB_QA_ACCESS_TOKEN'] end def remote_grid diff --git a/qa/qa/runtime/feature.rb b/qa/qa/runtime/feature.rb new file mode 100644 index 00000000000..1b4ae7adbbe --- /dev/null +++ b/qa/qa/runtime/feature.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module QA + module Runtime + module Feature + extend self + extend Support::Api + + SetFeatureError = Class.new(RuntimeError) + + def enable(key) + QA::Runtime::Logger.info("Enabling feature: #{key}") + set_feature(key, true) + end + + def disable(key) + QA::Runtime::Logger.info("Disabling feature: #{key}") + set_feature(key, false) + end + + private + + def api_client + @api_client ||= Runtime::API::Client.new(:gitlab) + end + + def set_feature(key, value) + request = Runtime::API::Request.new(api_client, "/features/#{key}") + response = post(request.url, { value: value }) + unless response.code == QA::Support::Api::HTTP_STATUS_CREATED + raise SetFeatureError, "Setting feature flag #{key} to #{value} failed with `#{response}`." + end + end + end + end +end diff --git a/qa/qa/scenario/bootable.rb b/qa/qa/scenario/bootable.rb index dd12ea6d492..038418be023 100644 --- a/qa/qa/scenario/bootable.rb +++ b/qa/qa/scenario/bootable.rb @@ -23,7 +23,7 @@ module QA arguments.parse!(argv) - self.perform(Runtime::Scenario.attributes, *arguments.default_argv) + self.perform(Runtime::Scenario.attributes, *argv) end private @@ -33,7 +33,13 @@ module QA end def options - @options ||= [] + # Scenario options/attributes are global. There's only ever one + # scenario at a time, but they can be inherited and we want scenarios + # to share the attributes of their ancestors. For example, `Mattermost` + # inherits from `Test::Instance::All` but if this were an instance + # variable then `Mattermost` wouldn't have access to the attributes + # in `All` + @@options ||= [] # rubocop:disable Style/ClassVars end def has_attributes? diff --git a/qa/qa/scenario/shared_attributes.rb b/qa/qa/scenario/shared_attributes.rb new file mode 100644 index 00000000000..40d5c6b1ff1 --- /dev/null +++ b/qa/qa/scenario/shared_attributes.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module QA + module Scenario + module SharedAttributes + include Bootable + + attribute :gitlab_address, '--address URL', 'Address of the instance to test' + attribute :enable_feature, '--enable-feature FEATURE_FLAG', 'Enable a feature before running tests' + end + end +end diff --git a/qa/qa/scenario/template.rb b/qa/qa/scenario/template.rb index cb1a1de6b9a..b8ea26e805e 100644 --- a/qa/qa/scenario/template.rb +++ b/qa/qa/scenario/template.rb @@ -18,19 +18,44 @@ module QA end end - def perform(address, *rspec_options) - Runtime::Scenario.define(:gitlab_address, address) + def perform(options, *args) + extract_address(:gitlab_address, options, args) ## # Perform before hooks, which are different for CE and EE # Runtime::Release.perform_before_hooks + Runtime::Feature.enable(options[:enable_feature]) if options.key?(:enable_feature) + Specs::Runner.perform do |specs| specs.tty = true specs.tags = self.class.focus - specs.options = rspec_options if rspec_options.any? + specs.options = args if args.any? end + ensure + Runtime::Feature.disable(options[:enable_feature]) if options.key?(:enable_feature) + end + + def extract_option(name, options, args) + option = if options.key?(name) + options[name] + else + args.shift + end + + Runtime::Scenario.define(name, option) + + option + end + + # For backwards-compatibility, if the gitlab instance address is not + # specified as an option parsed by OptionParser, it can be specified as + # the first argument + def extract_address(name, options, args) + address = extract_option(name, options, args) + + raise ::ArgumentError, "The address provided for `#{name}` is not valid: #{address}" unless Runtime::Address.valid?(address) end end end diff --git a/qa/qa/scenario/test/instance/all.rb b/qa/qa/scenario/test/instance/all.rb index a07c26431bd..168ac4c09a1 100644 --- a/qa/qa/scenario/test/instance/all.rb +++ b/qa/qa/scenario/test/instance/all.rb @@ -8,6 +8,7 @@ module QA module Instance class All < Template include Bootable + include SharedAttributes end end end diff --git a/qa/qa/scenario/test/instance/smoke.rb b/qa/qa/scenario/test/instance/smoke.rb index a7d2cb27f27..43f0623483e 100644 --- a/qa/qa/scenario/test/instance/smoke.rb +++ b/qa/qa/scenario/test/instance/smoke.rb @@ -8,6 +8,7 @@ module QA # class Smoke < Template include Bootable + include SharedAttributes tags :smoke end diff --git a/qa/qa/scenario/test/integration/mattermost.rb b/qa/qa/scenario/test/integration/mattermost.rb index ece6fba75c9..f5072ee227c 100644 --- a/qa/qa/scenario/test/integration/mattermost.rb +++ b/qa/qa/scenario/test/integration/mattermost.rb @@ -9,10 +9,13 @@ module QA class Mattermost < Test::Instance::All tags :mattermost - def perform(address, mattermost, *rspec_options) - Runtime::Scenario.define(:mattermost_address, mattermost) + attribute :mattermost_address, '--mattermost-address URL', 'Address of the Mattermost server' - super(address, *rspec_options) + def perform(options, *args) + extract_address(:gitlab_address, options, args) + extract_address(:mattermost_address, options, args) + + super(options, *args) end end end diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb index d8609aa037a..d1747280227 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb @@ -14,7 +14,7 @@ module QA end project.visit! - Page::Project::Menu.perform(&:click_members_settings) + Page::Project::Menu.perform(&:go_to_members_settings) Page::Project::Settings::Members.perform do |page| page.add_member(user.username) end diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/view_project_activity_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/view_project_activity_spec.rb index d4cedc9362d..fe92fbd3ffe 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/project/view_project_activity_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/project/view_project_activity_spec.rb @@ -14,8 +14,8 @@ module QA end project_push.project.visit! - Page::Project::Menu.perform(&:go_to_activity) - Page::Project::Activity.perform(&:go_to_push_events) + Page::Project::Menu.perform(&:click_activity) + Page::Project::Activity.perform(&:click_push_events) expect(page).to have_content('pushed new branch master') end diff --git a/qa/qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb b/qa/qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb index 7145b950b6c..358ab04eadc 100644 --- a/qa/qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb +++ b/qa/qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb @@ -5,15 +5,6 @@ module QA describe 'Issue creation' do let(:issue_title) { 'issue title' } - def create_issue - Runtime::Browser.visit(:gitlab, Page::Main::Login) - Page::Main::Login.act { sign_in_using_credentials } - - Resource::Issue.fabricate! do |issue| - issue.title = issue_title - end - end - it 'user creates an issue' do create_issue @@ -46,6 +37,15 @@ module QA end end end + + def create_issue + Runtime::Browser.visit(:gitlab, Page::Main::Login) + Page::Main::Login.act { sign_in_using_credentials } + + Resource::Issue.fabricate! do |issue| + issue.title = issue_title + end + end end end end diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb index c06f13ee204..1eada4a6c28 100644 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb @@ -3,7 +3,29 @@ module QA context 'Create' do describe 'Merge request creation' do - it 'user creates a new merge request' do + it 'user creates a new merge request', :smoke do + Runtime::Browser.visit(:gitlab, Page::Main::Login) + Page::Main::Login.act { sign_in_using_credentials } + + current_project = Resource::Project.fabricate! do |project| + project.name = 'project-with-merge-request' + end + + merge_request_title = 'This is a merge request' + merge_request_description = 'Great feature' + + Resource::MergeRequest.fabricate! do |merge_request| + merge_request.title = merge_request_title + merge_request.description = merge_request_description + merge_request.project = current_project + end + + expect(page).to have_content(merge_request_title) + expect(page).to have_content(merge_request_description) + expect(page).to have_content(/Opened [\w\s]+ ago/) + end + + it 'user creates a new merge request with a milestone and label' do gitlab_account_username = "@#{Runtime::User.username}" Runtime::Browser.visit(:gitlab, Page::Main::Login) @@ -24,9 +46,12 @@ module QA label.description = 'Merge Request label' end + merge_request_title = 'This is a merge request with a milestone and a label' + merge_request_description = 'Great feature with milestone' + Resource::MergeRequest.fabricate! do |merge_request| - merge_request.title = 'This is a merge request with a milestone' - merge_request.description = 'Great feature with milestone' + merge_request.title = merge_request_title + merge_request.description = merge_request_description merge_request.project = current_project merge_request.milestone = current_milestone merge_request.assignee = 'me' @@ -34,8 +59,8 @@ module QA end Page::MergeRequest::Show.perform do |merge_request| - expect(merge_request).to have_content('This is a merge request with a milestone') - expect(merge_request).to have_content('Great feature with milestone') + expect(merge_request).to have_content(merge_request_title) + expect(merge_request).to have_content(merge_request_description) expect(merge_request).to have_content(/Opened [\w\s]+ ago/) expect(merge_request).to have_assignee(gitlab_account_username) expect(merge_request).to have_label(new_label.title) @@ -47,25 +72,4 @@ module QA end end end - - describe 'creates a merge request', :smoke do - it 'user creates a new merge request' do - Runtime::Browser.visit(:gitlab, Page::Main::Login) - Page::Main::Login.act { sign_in_using_credentials } - - current_project = Resource::Project.fabricate! do |project| - project.name = 'project-with-merge-request' - end - - Resource::MergeRequest.fabricate! do |merge_request| - merge_request.title = 'This is a merge request' - merge_request.description = 'Great feature' - merge_request.project = current_project - end - - expect(page).to have_content('This is a merge request') - expect(page).to have_content('Great feature') - expect(page).to have_content(/Opened [\w\s]+ ago/) - end - end end diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb index 3fbcd77dac6..7a36f9ea420 100644 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb @@ -12,7 +12,7 @@ module QA end project.visit! - Page::Project::Menu.perform(&:go_to_settings) + Page::Project::Menu.perform(&:click_settings) Page::Project::Settings::MergeRequest.perform(&:enable_ff_only) merge_request = Resource::MergeRequest.fabricate! do |merge_request| diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb index 8e181eb28c6..daeee665c93 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb @@ -57,7 +57,7 @@ module QA end it 'branches are correctly listed after CRUD operations' do - Page::Project::Menu.perform(&:click_repository_branches) + Page::Project::Menu.perform(&:go_to_repository_branches) expect(page).to have_content(master_branch) expect(page).to have_content(second_branch) diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/add_ssh_key_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/add_ssh_key_spec.rb index ff879fdeb16..f41240b7605 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/add_ssh_key_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/add_ssh_key_spec.rb @@ -16,7 +16,7 @@ module QA expect(page).to have_content("Title: #{key_title}") expect(page).to have_content(key.fingerprint) - Page::Main::Menu.act { go_to_profile_settings } + Page::Main::Menu.act { click_settings_link } Page::Profile::Menu.act { click_ssh_keys } Page::Profile::SSHKeys.perform do |ssh_keys| diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/protocol_v2_push_ssh_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/protocol_v2_push_ssh_spec.rb index eb59b54e9ab..dc42191b448 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/protocol_v2_push_ssh_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/protocol_v2_push_ssh_spec.rb @@ -31,7 +31,7 @@ module QA # Remove the SSH key login - Page::Main::Menu.perform(&:go_to_profile_settings) + Page::Main::Menu.perform(&:click_settings_link) Page::Profile::Menu.perform(&:click_ssh_keys) Page::Profile::SSHKeys.perform do |ssh_keys| ssh_keys.remove_key(key_title) diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/push_mirroring_over_http_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/push_mirroring_over_http_spec.rb index 99601e3d230..23008a58af8 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/push_mirroring_over_http_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/push_mirroring_over_http_spec.rb @@ -21,7 +21,7 @@ module QA end source_project_push.project.visit! - Page::Project::Menu.perform(&:click_repository_settings) + Page::Project::Menu.perform(&:go_to_repository_settings) Page::Project::Settings::Repository.perform do |settings| settings.expand_mirroring_repositories do |mirror_settings| # Configure the source project to push to the target project diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/push_over_http_file_size_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/push_over_http_file_size_spec.rb index 243f0b83b77..a544efb35ee 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/push_over_http_file_size_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/push_over_http_file_size_spec.rb @@ -40,12 +40,12 @@ module QA set_file_size_limit(1) expect(page).to have_content("Application settings saved successfully") - push = push_new_file('oversize_file_2.bin', wait_for_push: false) - expect(push.output).to have_content 'remote: fatal: pack exceeds maximum allowed size' + expect { push_new_file('oversize_file_2.bin', wait_for_push: false) } + .to raise_error(QA::Git::Repository::RepositoryCommandError, /remote: fatal: pack exceeds maximum allowed size/) end def set_file_size_limit(limit) - Page::Main::Menu.perform(&:go_to_admin_area) + Page::Main::Menu.perform(&:click_admin_area) Page::Admin::Menu.perform(&:go_to_general_settings) Page::Admin::Settings::General.perform do |setting| @@ -65,6 +65,7 @@ module QA p.file_content = SecureRandom.random_bytes(2000000) p.commit_message = 'Adding a new file' p.wait_for_push = wait_for_push + p.new_branch = false end end end diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/use_ssh_key_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/use_ssh_key_spec.rb index 7223831d96f..a0251e1c385 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/use_ssh_key_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/use_ssh_key_spec.rb @@ -28,7 +28,7 @@ module QA expect(page).to have_content('README.md') expect(page).to have_content('Test Use SSH Key') - Page::Main::Menu.perform(&:go_to_profile_settings) + Page::Main::Menu.perform(&:click_settings_link) Page::Profile::Menu.perform(&:click_ssh_keys) Page::Profile::SSHKeys.perform do |ssh_keys| diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/user_views_commit_diff_patch_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/user_views_commit_diff_patch_spec.rb index b862a7bd1ed..b7400cdca97 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/user_views_commit_diff_patch_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/user_views_commit_diff_patch_spec.rb @@ -33,7 +33,7 @@ module QA def view_commit @project.visit! Page::Project::Show.perform do |page| - page.go_to_commit(@commit_message) + page.click_commit(@commit_message) end end diff --git a/qa/qa/specs/features/browser_ui/3_create/snippet/create_snippet_spec.rb b/qa/qa/specs/features/browser_ui/3_create/snippet/create_snippet_spec.rb index ab53dff464e..b643468a664 100644 --- a/qa/qa/specs/features/browser_ui/3_create/snippet/create_snippet_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/snippet/create_snippet_spec.rb @@ -7,7 +7,7 @@ module QA Runtime::Browser.visit(:gitlab, Page::Main::Login) Page::Main::Login.perform(&:sign_in_using_credentials) - Page::Main::Menu.perform(&:go_to_snippets) + Page::Main::Menu.perform(&:click_snippets_link) Resource::Snippet.fabricate_via_browser_ui! do |snippet| snippet.title = 'Snippet title' diff --git a/qa/qa/specs/features/browser_ui/3_create/wiki/create_edit_clone_push_wiki_spec.rb b/qa/qa/specs/features/browser_ui/3_create/wiki/create_edit_clone_push_wiki_spec.rb index 29589ec870a..309ae6cd986 100644 --- a/qa/qa/specs/features/browser_ui/3_create/wiki/create_edit_clone_push_wiki_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/wiki/create_edit_clone_push_wiki_spec.rb @@ -20,7 +20,7 @@ module QA validate_content('My First Wiki Content') - Page::Project::Wiki::Edit.perform(&:go_to_edit_page) + Page::Project::Wiki::Edit.perform(&:click_edit) Page::Project::Wiki::New.perform do |page| page.set_content("My Second Wiki Content") page.save_changes diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb index 2238d6c382e..7c1d4489c47 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb @@ -70,7 +70,7 @@ module QA puts 'Waiting for the runner to process the pipeline' sleep 15 # Runner should process all jobs within 15 seconds. - Page::Project::Pipeline::Index.perform(&:go_to_latest_pipeline) + Page::Project::Pipeline::Index.perform(&:click_on_latest_pipeline) Page::Project::Pipeline::Show.perform do |pipeline| expect(pipeline).to be_running diff --git a/qa/qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb b/qa/qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb index 3f65eabc756..609155da855 100644 --- a/qa/qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb +++ b/qa/qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb @@ -91,8 +91,8 @@ module QA sha1sum = Digest::SHA1.hexdigest(gitlab_ci) Page::Project::Menu.perform(&:click_ci_cd_pipelines) - Page::Project::Pipeline::Index.perform(&:go_to_latest_pipeline) - Page::Project::Pipeline::Show.perform(&:go_to_first_job) + Page::Project::Pipeline::Index.perform(&:click_on_latest_pipeline) + Page::Project::Pipeline::Show.perform(&:click_on_first_job) Page::Project::Job::Show.perform do |job| expect(job).to be_successful diff --git a/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb b/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb index 16aca7c0e22..bb1f775da75 100644 --- a/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb +++ b/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb @@ -63,10 +63,10 @@ module QA end Page::Project::Menu.perform(&:click_ci_cd_pipelines) - Page::Project::Pipeline::Index.perform(&:go_to_latest_pipeline) + Page::Project::Pipeline::Index.perform(&:click_on_latest_pipeline) Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.go_to_job('build') + pipeline.click_job('build') end Page::Project::Job::Show.perform do |job| expect(job).to be_successful(timeout: 600) @@ -75,7 +75,7 @@ module QA end Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.go_to_job('test') + pipeline.click_job('test') end Page::Project::Job::Show.perform do |job| expect(job).to be_successful(timeout: 600) @@ -84,7 +84,7 @@ module QA end Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.go_to_job('production') + pipeline.click_job('production') end Page::Project::Job::Show.perform do |job| expect(job).to be_successful(timeout: 1200) @@ -92,9 +92,9 @@ module QA job.click_element(:pipeline_path) end - Page::Project::Menu.perform(&:click_operations_environments) + Page::Project::Menu.perform(&:go_to_operations_environments) Page::Project::Operations::Environments::Index.perform do |index| - index.go_to_environment('production') + index.click_environment_link('production') end Page::Project::Operations::Environments::Show.perform do |show| show.view_deployment do @@ -106,8 +106,7 @@ module QA end end - # Failure issue: https://gitlab.com/gitlab-org/quality/nightly/issues/87 - describe 'Auto DevOps', :smoke, :quarantine do + describe 'Auto DevOps', :smoke do it 'enables AutoDevOps by default' do login @@ -116,12 +115,6 @@ module QA p.description = 'Project with AutoDevOps' end - project.visit! - - Page::Alert::AutoDevopsAlert.perform do |alert| - expect(alert).to have_text(/.*The Auto DevOps pipeline has been enabled.*/) - end - # Create AutoDevOps repo Resource::Repository::ProjectPush.fabricate! do |push| push.project = project @@ -132,7 +125,7 @@ module QA end Page::Project::Menu.perform(&:click_ci_cd_pipelines) - Page::Project::Pipeline::Index.perform(&:go_to_latest_pipeline) + Page::Project::Pipeline::Index.perform(&:click_on_latest_pipeline) Page::Project::Pipeline::Show.perform do |pipeline| expect(pipeline).to have_tag('Auto DevOps') diff --git a/qa/qa/specs/features/browser_ui/7_configure/mattermost/create_group_with_mattermost_team_spec.rb b/qa/qa/specs/features/browser_ui/7_configure/mattermost/create_group_with_mattermost_team_spec.rb index 7096864e011..8383dcdb983 100644 --- a/qa/qa/specs/features/browser_ui/7_configure/mattermost/create_group_with_mattermost_team_spec.rb +++ b/qa/qa/specs/features/browser_ui/7_configure/mattermost/create_group_with_mattermost_team_spec.rb @@ -9,7 +9,7 @@ module QA Page::Main::Menu.act { go_to_groups } Page::Dashboard::Groups.perform do |page| - page.go_to_new_group + page.click_new_group expect(page).to have_content( /Create a Mattermost team for this group/ diff --git a/qa/qa/support/api.rb b/qa/qa/support/api.rb index 229bfb44fa5..31cff5a241c 100644 --- a/qa/qa/support/api.rb +++ b/qa/qa/support/api.rb @@ -1,6 +1,9 @@ module QA module Support module Api + HTTP_STATUS_OK = 200 + HTTP_STATUS_CREATED = 201 + def post(url, payload) RestClient::Request.execute( method: :post, diff --git a/qa/qa/tools/delete_subgroups.rb b/qa/qa/tools/delete_subgroups.rb index c5c48e77ade..3f752adbe6f 100644 --- a/qa/qa/tools/delete_subgroups.rb +++ b/qa/qa/tools/delete_subgroups.rb @@ -3,7 +3,7 @@ require_relative '../../qa' # This script deletes all subgroups of a group specified by ENV['GROUP_NAME_OR_PATH'] -# Required environment variables: PERSONAL_ACCESS_TOKEN and GITLAB_ADDRESS +# Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS # Optional environment variable: GROUP_NAME_OR_PATH (defaults to 'gitlab-qa-sandbox-group') # Run `rake delete_subgroups` @@ -14,9 +14,9 @@ module QA def initialize raise ArgumentError, "Please provide GITLAB_ADDRESS" unless ENV['GITLAB_ADDRESS'] - raise ArgumentError, "Please provide PERSONAL_ACCESS_TOKEN" unless ENV['PERSONAL_ACCESS_TOKEN'] + raise ArgumentError, "Please provide GITLAB_QA_ACCESS_TOKEN" unless ENV['GITLAB_QA_ACCESS_TOKEN'] - @api_client = Runtime::API::Client.new(ENV['GITLAB_ADDRESS'], personal_access_token: ENV['PERSONAL_ACCESS_TOKEN']) + @api_client = Runtime::API::Client.new(ENV['GITLAB_ADDRESS'], personal_access_token: ENV['GITLAB_QA_ACCESS_TOKEN']) end def run diff --git a/qa/qa/tools/generate_perf_testdata.rb b/qa/qa/tools/generate_perf_testdata.rb index ad515014794..49a1af8e9f0 100644 --- a/qa/qa/tools/generate_perf_testdata.rb +++ b/qa/qa/tools/generate_perf_testdata.rb @@ -2,9 +2,10 @@ require 'securerandom' require 'faker' +require 'yaml' require_relative '../../qa' # This script generates testdata for Performance Testing. -# Required environment variables: PERSONAL_ACCESS_TOKEN and GITLAB_ADDRESS +# Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS # This job creates a urls.txt which contains a hash of all the URLs needed for Performance Testing # Run `rake generate_perf_testdata` @@ -15,12 +16,13 @@ module QA def initialize raise ArgumentError, "Please provide GITLAB_ADDRESS" unless ENV['GITLAB_ADDRESS'] - raise ArgumentError, "Please provide PERSONAL_ACCESS_TOKEN" unless ENV['PERSONAL_ACCESS_TOKEN'] + raise ArgumentError, "Please provide GITLAB_QA_ACCESS_TOKEN" unless ENV['GITLAB_QA_ACCESS_TOKEN'] - @api_client = Runtime::API::Client.new(ENV['GITLAB_ADDRESS'], personal_access_token: ENV['PERSONAL_ACCESS_TOKEN']) + @api_client = Runtime::API::Client.new(ENV['GITLAB_ADDRESS'], personal_access_token: ENV['GITLAB_QA_ACCESS_TOKEN']) @group_name = "gitlab-qa-perf-sandbox-#{SecureRandom.hex(8)}" @project_name = "my-test-project-#{SecureRandom.hex(8)}" - @urls = {} + @visibility = "public" + @urls = { host: ENV['GITLAB_ADDRESS'] } end def run @@ -39,26 +41,26 @@ module QA threads_arr = [] methods_arr.each do |m| - threads_arr << Thread.new {m.call} + threads_arr << Thread.new { m.call } end threads_arr.each(&:join) STDOUT.puts "\nURLs: #{@urls}" - File.open("urls.txt", "w") { |file| file.puts @urls.to_s} + File.open("urls.yml", "w") { |file| file.puts @urls.stringify_keys.to_yaml } STDOUT.puts "\nDone" end private def create_group - group_search_response = post Runtime::API::Request.new(@api_client, "/groups").url, "name=#{@group_name}&path=#{@group_name}" + group_search_response = post Runtime::API::Request.new(@api_client, "/groups").url, "name=#{@group_name}&path=#{@group_name}&visibility=#{@visibility}" group = JSON.parse(group_search_response.body) @urls[:group_page] = group["web_url"] group["id"] end def create_project(group_id) - create_project_response = post Runtime::API::Request.new(@api_client, "/projects").url, "name=#{@project_name}&namespace_id=#{group_id}" + create_project_response = post Runtime::API::Request.new(@api_client, "/projects").url, "name=#{@project_name}&namespace_id=#{group_id}&visibility=#{@visibility}" @urls[:project_page] = JSON.parse(create_project_response.body)["web_url"] end diff --git a/qa/qa/tools/revoke_all_personal_access_tokens.rb b/qa/qa/tools/revoke_all_personal_access_tokens.rb index 7484b633bf6..8a74f52324d 100644 --- a/qa/qa/tools/revoke_all_personal_access_tokens.rb +++ b/qa/qa/tools/revoke_all_personal_access_tokens.rb @@ -27,7 +27,7 @@ module QA Runtime::Browser.visit(ENV['GITLAB_ADDRESS'], Page::Main::Login) Page::Main::Login.perform(&:sign_in_using_credentials) - Page::Main::Menu.perform(&:go_to_profile_settings) + Page::Main::Menu.perform(&:click_settings_link) Page::Profile::Menu.perform(&:click_access_tokens) token_name = 'api-test-token' diff --git a/qa/spec/runtime/env_spec.rb b/qa/spec/runtime/env_spec.rb index fc51f45c3a1..04085efe2ce 100644 --- a/qa/spec/runtime/env_spec.rb +++ b/qa/spec/runtime/env_spec.rb @@ -90,13 +90,13 @@ describe QA::Runtime::Env do described_class.instance_variable_set(:@personal_access_token, nil) end - context 'when PERSONAL_ACCESS_TOKEN is set' do + context 'when GITLAB_QA_ACCESS_TOKEN is set' do before do - stub_env('PERSONAL_ACCESS_TOKEN', 'a_token') + stub_env('GITLAB_QA_ACCESS_TOKEN', 'a_token_too') end it 'returns specified token from env' do - expect(described_class.personal_access_token).to eq 'a_token' + expect(described_class.personal_access_token).to eq 'a_token_too' end end diff --git a/qa/spec/runtime/feature_spec.rb b/qa/spec/runtime/feature_spec.rb new file mode 100644 index 00000000000..192299b7857 --- /dev/null +++ b/qa/spec/runtime/feature_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +describe QA::Runtime::Feature do + let(:api_client) { double('QA::Runtime::API::Client') } + let(:request) { Struct.new(:url).new('http://api') } + let(:response) { Struct.new(:code).new(201) } + + before do + allow(described_class).to receive(:api_client).and_return(api_client) + end + + describe '.enable' do + it 'enables a feature flag' do + expect(QA::Runtime::API::Request) + .to receive(:new) + .with(api_client, "/features/a-flag") + .and_return(request) + expect(described_class) + .to receive(:post) + .with(request.url, { value: true }) + .and_return(response) + + subject.enable('a-flag') + end + end + + describe '.disable' do + it 'disables a feature flag' do + expect(QA::Runtime::API::Request) + .to receive(:new) + .with(api_client, "/features/a-flag") + .and_return(request) + expect(described_class) + .to receive(:post) + .with(request.url, { value: false }) + .and_return(response) + + subject.disable('a-flag') + end + end +end diff --git a/qa/spec/runtime/scenario_spec.rb b/qa/spec/runtime/scenario_spec.rb index 7009192bcc0..70fc71ffc02 100644 --- a/qa/spec/runtime/scenario_spec.rb +++ b/qa/spec/runtime/scenario_spec.rb @@ -13,6 +13,14 @@ describe QA::Runtime::Scenario do .to eq(my_attribute: 'some-value', another_attribute: 'another-value') end + it 'replaces an existing attribute' do + subject.define(:my_attribute, 'some-value') + subject.define(:my_attribute, 'another-value') + + expect(subject.my_attribute).to eq 'another-value' + expect(subject.attributes).to eq(my_attribute: 'another-value') + end + it 'raises error when attribute is not known' do expect { subject.invalid_accessor } .to raise_error ArgumentError, /invalid_accessor/ diff --git a/qa/spec/scenario/bootable_spec.rb b/qa/spec/scenario/bootable_spec.rb index 273aac7677e..bd89b21f7fb 100644 --- a/qa/spec/scenario/bootable_spec.rb +++ b/qa/spec/scenario/bootable_spec.rb @@ -4,14 +4,21 @@ describe QA::Scenario::Bootable do .include(described_class) end + before do + allow(subject).to receive(:options).and_return([]) + allow(QA::Runtime::Scenario).to receive(:attributes).and_return({}) + end + it 'makes it possible to define the scenario attribute' do subject.class_eval do attribute :something, '--something SOMETHING', 'Some attribute' attribute :another, '--another ANOTHER', 'Some other attribute' end + # If we run just this test from the command line it fails unless + # we include the command line args that we use to select this test. expect(subject).to receive(:perform) - .with(something: 'test', another: 'other') + .with({ something: 'test', another: 'other' }) subject.launch!(%w[--another other --something test]) end diff --git a/qa/spec/scenario/template_spec.rb b/qa/spec/scenario/template_spec.rb new file mode 100644 index 00000000000..f97fc22daf9 --- /dev/null +++ b/qa/spec/scenario/template_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +describe QA::Scenario::Template do + let(:feature) { spy('Runtime::Feature') } + let(:release) { spy('Runtime::Release') } + + before do + stub_const('QA::Runtime::Release', release) + stub_const('QA::Runtime::Feature', feature) + allow(QA::Specs::Runner).to receive(:perform) + allow(QA::Runtime::Address).to receive(:valid?).and_return(true) + end + + it 'allows a feature to be enabled' do + subject.perform({ enable_feature: 'a-feature' }) + + expect(feature).to have_received(:enable).with('a-feature') + end + + it 'ensures an enabled feature is disabled afterwards' do + allow(QA::Specs::Runner).to receive(:perform).and_raise('failed test') + + expect { subject.perform({ enable_feature: 'a-feature' }) }.to raise_error('failed test') + + expect(feature).to have_received(:enable).with('a-feature') + expect(feature).to have_received(:disable).with('a-feature') + end +end diff --git a/qa/spec/scenario/test/integration/github_spec.rb b/qa/spec/scenario/test/integration/github_spec.rb index c2aeb1ded1d..6112ba7c694 100644 --- a/qa/spec/scenario/test/integration/github_spec.rb +++ b/qa/spec/scenario/test/integration/github_spec.rb @@ -12,7 +12,7 @@ describe QA::Scenario::Test::Integration::Github do let(:tags) { [:github] } it 'requires a GitHub access token' do - subject.perform('gitlab_address') + subject.perform(args) expect(env).to have_received(:require_github_access_token!) end diff --git a/qa/spec/scenario/test/integration/mattermost_spec.rb b/qa/spec/scenario/test/integration/mattermost_spec.rb index 59caf2ba2cd..4e75e72f4d2 100644 --- a/qa/spec/scenario/test/integration/mattermost_spec.rb +++ b/qa/spec/scenario/test/integration/mattermost_spec.rb @@ -4,14 +4,21 @@ describe QA::Scenario::Test::Integration::Mattermost do context '#perform' do it_behaves_like 'a QA scenario class' do let(:args) { %w[gitlab_address mattermost_address] } + let(:args) do + { + gitlab_address: 'http://gitlab_address', + mattermost_address: 'http://mattermost_address' + } + end + let(:named_options) { %w[--address http://gitlab_address --mattermost-address http://mattermost_address] } let(:tags) { [:mattermost] } let(:options) { ['path1']} it 'requires a GitHub access token' do - subject.perform(*args) + subject.perform(args) expect(attributes).to have_received(:define) - .with(:mattermost_address, 'mattermost_address') + .with(:mattermost_address, 'http://mattermost_address') end end end diff --git a/qa/spec/shared_examples/scenario_shared_examples.rb b/qa/spec/shared_examples/scenario_shared_examples.rb index 5fd55d7d96b..697e6cb39c8 100644 --- a/qa/spec/shared_examples/scenario_shared_examples.rb +++ b/qa/spec/shared_examples/scenario_shared_examples.rb @@ -2,19 +2,23 @@ shared_examples 'a QA scenario class' do let(:attributes) { spy('Runtime::Scenario') } - let(:release) { spy('Runtime::Release') } let(:runner) { spy('Specs::Runner') } + let(:release) { spy('Runtime::Release') } + let(:feature) { spy('Runtime::Feature') } - let(:args) { ['gitlab_address'] } + let(:args) { { gitlab_address: 'http://gitlab_address' } } + let(:named_options) { %w[--address http://gitlab_address] } let(:tags) { [] } let(:options) { %w[path1 path2] } before do + stub_const('QA::Specs::Runner', runner) stub_const('QA::Runtime::Release', release) stub_const('QA::Runtime::Scenario', attributes) - stub_const('QA::Specs::Runner', runner) + stub_const('QA::Runtime::Feature', feature) allow(runner).to receive(:perform).and_yield(runner) + allow(QA::Runtime::Address).to receive(:valid?).and_return(true) end it 'responds to perform' do @@ -22,28 +26,48 @@ shared_examples 'a QA scenario class' do end it 'sets an address of the subject' do - subject.perform(*args) + subject.perform(args) - expect(attributes).to have_received(:define).with(:gitlab_address, 'gitlab_address') + expect(attributes).to have_received(:define).with(:gitlab_address, 'http://gitlab_address').at_least(:once) end it 'performs before hooks' do - subject.perform(*args) + subject.perform(args) expect(release).to have_received(:perform_before_hooks) end it 'sets tags on runner' do - subject.perform(*args) + subject.perform(args) expect(runner).to have_received(:tags=).with(tags) end context 'specifying RSpec options' do it 'sets options on runner' do - subject.perform(*args, *options) + subject.perform(args, *options) expect(runner).to have_received(:options=).with(options) end end + + context 'with named command-line options' do + it 'converts options to attributes' do + described_class.launch!(named_options) + + args do |k, v| + expect(attributes).to have_received(:define).with(k, v) + end + end + + it 'raises an error if the option is invalid' do + expect { described_class.launch!(['--foo']) }.to raise_error(OptionParser::InvalidOption) + end + + it 'passes on options after --' do + expect(described_class).to receive(:perform).with(attributes, *%w[--tag quarantine]) + + described_class.launch!(named_options.push(*%w[-- --tag quarantine])) + end + end end |