diff options
Diffstat (limited to 'qa')
21 files changed, 281 insertions, 65 deletions
diff --git a/qa/.rspec_parallel b/qa/.rspec_parallel new file mode 100644 index 00000000000..e5927927eaa --- /dev/null +++ b/qa/.rspec_parallel @@ -0,0 +1,5 @@ +--color +--format documentation +--format ParallelTests::RSpec::SummaryLogger --out tmp/spec_summary.log +--format ParallelTests::RSpec::RuntimeLogger --out tmp/parallel_runtime_rspec.log +--require spec_helper diff --git a/qa/Gemfile b/qa/Gemfile index 12994b85322..c46be8a0362 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -1,5 +1,6 @@ source 'https://rubygems.org' +gem 'gitlab-qa' gem 'pry-byebug', '~> 3.5.1', platform: :mri gem 'capybara', '~> 2.16.1' gem 'capybara-screenshot', '~> 1.0.18' @@ -11,3 +12,4 @@ gem 'nokogiri', '~> 1.10.3' gem 'rspec-retry', '~> 0.6.1' gem 'faker', '~> 1.6', '>= 1.6.6' gem 'knapsack', '~> 1.17' +gem 'parallel_tests', '~> 2.29' diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 6b0635ed0e2..73aabf2c6ad 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -35,6 +35,7 @@ GEM faker (1.9.3) i18n (>= 0.7) ffi (1.9.25) + gitlab-qa (4.0.0) http-cookie (1.0.3) domain_name (~> 0.5) i18n (0.9.1) @@ -53,6 +54,9 @@ GEM netrc (0.11.0) nokogiri (1.10.3) mini_portile2 (~> 2.4.0) + parallel (1.17.0) + parallel_tests (2.29.0) + parallel pry (0.11.3) coderay (~> 1.1.0) method_source (~> 0.9.0) @@ -104,8 +108,10 @@ DEPENDENCIES capybara (~> 2.16.1) capybara-screenshot (~> 1.0.18) faker (~> 1.6, >= 1.6.6) + gitlab-qa knapsack (~> 1.17) nokogiri (~> 1.10.3) + parallel_tests (~> 2.29) pry-byebug (~> 3.5.1) rake (~> 12.3.0) rspec (~> 3.7) @@ -360,6 +360,7 @@ module QA module Specs autoload :Config, 'qa/specs/config' autoload :Runner, 'qa/specs/runner' + autoload :ParallelRunner, 'qa/specs/parallel_runner' module Helpers autoload :Quarantine, 'qa/specs/helpers/quarantine' diff --git a/qa/qa/resource/issue.rb b/qa/qa/resource/issue.rb index 9c57a0f5afb..51b2af8b4ef 100644 --- a/qa/qa/resource/issue.rb +++ b/qa/qa/resource/issue.rb @@ -16,6 +16,10 @@ module QA attribute :labels attribute :title + def initialize + @labels = [] + end + def fabricate! project.visit! @@ -38,7 +42,7 @@ module QA def api_post_body { - labels: [labels], + labels: labels, title: title } end diff --git a/qa/qa/runtime/browser.rb b/qa/qa/runtime/browser.rb index ed0779b93cc..2987bb1a213 100644 --- a/qa/qa/runtime/browser.rb +++ b/qa/qa/runtime/browser.rb @@ -13,6 +13,8 @@ module QA NotRespondingError = Class.new(RuntimeError) + CAPYBARA_MAX_WAIT_TIME = 10 + def initialize self.class.configure! end @@ -43,6 +45,8 @@ module QA end end + Capybara.server_port = 9887 + ENV['TEST_ENV_NUMBER'].to_i + return if Capybara.drivers.include?(:chrome) Capybara.register_driver QA::Runtime::Env.browser do |app| @@ -119,7 +123,7 @@ module QA Capybara.configure do |config| config.default_driver = QA::Runtime::Env.browser config.javascript_driver = QA::Runtime::Env.browser - config.default_max_wait_time = 10 + config.default_max_wait_time = CAPYBARA_MAX_WAIT_TIME # https://github.com/mattheworiordan/capybara-screenshot/issues/164 config.save_path = ::File.expand_path('../../tmp', __dir__) end diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb index 96f337dc081..d50f618ff82 100644 --- a/qa/qa/runtime/env.rb +++ b/qa/qa/runtime/env.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'gitlab/qa' + module QA module Runtime module Env @@ -7,6 +9,8 @@ module QA attr_writer :personal_access_token, :ldap_username, :ldap_password + ENV_VARIABLES = Gitlab::QA::Runtime::Env::ENV_VARIABLES + # The environment variables used to indicate if the environment under test # supports the given feature SUPPORTED_FEATURES = { @@ -201,6 +205,10 @@ module QA enabled?(ENV[SUPPORTED_FEATURES[feature]], default: true) end + def runtime_scenario_attributes + ENV['QA_RUNTIME_SCENARIO_ATTRIBUTES'] + end + private def remote_grid_credentials diff --git a/qa/qa/runtime/scenario.rb b/qa/qa/runtime/scenario.rb index 5067322804b..3662ebe671b 100644 --- a/qa/qa/runtime/scenario.rb +++ b/qa/qa/runtime/scenario.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'json' + module QA module Runtime ## @@ -24,6 +26,10 @@ module QA end end + def from_env(var) + JSON.parse(Runtime::Env.runtime_scenario_attributes).each { |k, v| define(k, v) } + end + def method_missing(name, *) raise ArgumentError, "Scenario attribute `#{name}` not defined!" end diff --git a/qa/qa/scenario/shared_attributes.rb b/qa/qa/scenario/shared_attributes.rb index 40d5c6b1ff1..52f50ec8c27 100644 --- a/qa/qa/scenario/shared_attributes.rb +++ b/qa/qa/scenario/shared_attributes.rb @@ -7,6 +7,7 @@ module QA attribute :gitlab_address, '--address URL', 'Address of the instance to test' attribute :enable_feature, '--enable-feature FEATURE_FLAG', 'Enable a feature before running tests' + attribute :parallel, '--parallel', 'Execute tests in parallel' end end end diff --git a/qa/qa/specs/features/browser_ui/2_plan/issue/check_mentions_for_xss_spec.rb b/qa/qa/specs/features/browser_ui/2_plan/issue/check_mentions_for_xss_spec.rb index 013cea0a40e..5eceeb9661c 100644 --- a/qa/qa/specs/features/browser_ui/2_plan/issue/check_mentions_for_xss_spec.rb +++ b/qa/qa/specs/features/browser_ui/2_plan/issue/check_mentions_for_xss_spec.rb @@ -9,12 +9,12 @@ module QA Runtime::Browser.visit(:gitlab, Page::Main::Login) Page::Main::Login.perform(&:sign_in_using_credentials) - user = Resource::User.fabricate! do |user| + user = Resource::User.fabricate_via_api! do |user| user.name = "eve <img src=x onerror=alert(2)<img src=x onerror=alert(1)>" user.password = "test1234" end - project = Resource::Project.fabricate! do |resource| + project = Resource::Project.fabricate_via_api! do |resource| resource.name = 'xss-test-for-mentions-project' end project.visit! @@ -24,10 +24,11 @@ module QA page.add_member(user.username) end - Resource::Issue.fabricate_via_browser_ui! do |issue| + issue = Resource::Issue.fabricate_via_api! do |issue| issue.title = issue_title issue.project = project end + issue.visit! Page::Project::Issue::Show.perform do |show_page| show_page.select_all_activities_filter diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/view_merge_request_diff_patch_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/view_merge_request_diff_patch_spec.rb index db33c6330ff..9e48ee7ca2a 100644 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/view_merge_request_diff_patch_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/view_merge_request_diff_patch_spec.rb @@ -1,8 +1,7 @@ # frozen_string_literal: true module QA - # https://gitlab.com/gitlab-org/quality/staging/issues/55 - context 'Create', :quarantine do + context 'Create' do describe 'Download merge request patch and diff' do before(:context) do Runtime::Browser.visit(:gitlab, Page::Main::Login) diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/push_mirroring_over_http_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/push_mirroring_over_http_spec.rb index 23008a58af8..448d4980727 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/push_mirroring_over_http_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/push_mirroring_over_http_spec.rb @@ -1,8 +1,7 @@ # frozen_string_literal: true module QA - # https://gitlab.com/gitlab-org/quality/staging/issues/40 - context 'Create', :quarantine do + context 'Create' do describe 'Push mirror a repository over HTTP' do it 'configures and syncs a (push) mirrored repository' do Runtime::Browser.visit(:gitlab, Page::Main::Login) diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/push_over_http_file_size_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/push_over_http_file_size_spec.rb index 247cde38e52..11fd570d131 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/push_over_http_file_size_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/push_over_http_file_size_spec.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true module QA - context 'Create', :requires_admin do + # Failure issue: https://gitlab.com/gitlab-org/quality/nightly/issues/113 + context 'Create', :requires_admin, :quarantine do describe 'push after setting the file size limit via admin/application_settings' do before(:context) do @project = Resource::Project.fabricate_via_api! do |p| diff --git a/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb b/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb index 5ca9ddb6b19..99f0838b864 100644 --- a/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb +++ b/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb @@ -9,6 +9,52 @@ module QA Page::Main::Login.perform(&:sign_in_using_credentials) end + def disable_optional_jobs(project) + # Disable code_quality check in Auto DevOps pipeline as it takes + # too long and times out the test + Resource::CiVariable.fabricate_via_api! do |resource| + resource.project = project + resource.key = 'CODE_QUALITY_DISABLED' + resource.value = '1' + resource.masked = false + end + + Resource::CiVariable.fabricate_via_api! do |resource| + resource.project = project + resource.key = 'LICENSE_MANAGEMENT_DISABLED' + resource.value = '1' + resource.masked = false + end + + Resource::CiVariable.fabricate_via_api! do |resource| + resource.project = project + resource.key = 'SAST_DISABLED' + resource.value = '1' + resource.masked = false + end + + Resource::CiVariable.fabricate_via_api! do |resource| + resource.project = project + resource.key = 'DEPENDENCY_SCANNING_DISABLED' + resource.value = '1' + resource.masked = false + end + + Resource::CiVariable.fabricate_via_api! do |resource| + resource.project = project + resource.key = 'CONTAINER_SCANNING_DISABLED' + resource.value = '1' + resource.masked = false + end + + Resource::CiVariable.fabricate_via_api! do |resource| + resource.project = project + resource.key = 'DAST_DISABLED' + resource.value = '1' + resource.masked = false + end + end + # Failure issue: https://gitlab.com/gitlab-org/quality/nightly/issues/108 describe 'Auto DevOps support', :orchestrated, :kubernetes, :quarantine do context 'when rbac is enabled' do @@ -28,14 +74,7 @@ module QA p.description = 'Project with Auto DevOps' end - # Disable code_quality check in Auto DevOps pipeline as it takes - # too long and times out the test - Resource::CiVariable.fabricate! do |resource| - resource.project = @project - resource.key = 'CODE_QUALITY_DISABLED' - resource.value = '1' - resource.masked = false - end + disable_optional_jobs(@project) # Set an application secret CI variable (prefixed with K8S_SECRET_) Resource::CiVariable.fabricate! do |resource| diff --git a/qa/qa/specs/parallel_runner.rb b/qa/qa/specs/parallel_runner.rb new file mode 100644 index 00000000000..b92fdb610b6 --- /dev/null +++ b/qa/qa/specs/parallel_runner.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'open3' + +module QA + module Specs + module ParallelRunner + module_function + + def run(args) + unless args.include?('--') + index = args.index { |opt| opt.include?('features') } + + args.insert(index, '--') if index + end + + env = {} + Runtime::Env::ENV_VARIABLES.each_key do |key| + env[key] = ENV[key] if ENV[key] + end + env['QA_RUNTIME_SCENARIO_ATTRIBUTES'] = Runtime::Scenario.attributes.to_json + env['GITLAB_QA_ACCESS_TOKEN'] = Runtime::API::Client.new(:gitlab).personal_access_token unless env['GITLAB_QA_ACCESS_TOKEN'] + + cmd = "bundle exec parallel_test -t rspec --combine-stderr --serialize-stdout -- #{args.flatten.join(' ')}" + ::Open3.popen2e(env, cmd) do |_, out, wait| + out.each { |line| puts line } + + exit wait.value.exitstatus + end + end + end + end +end diff --git a/qa/qa/specs/runner.rb b/qa/qa/specs/runner.rb index f1cb9378de8..6aa08cf77b4 100644 --- a/qa/qa/specs/runner.rb +++ b/qa/qa/specs/runner.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true +require 'knapsack' require 'rspec/core' require 'rspec/expectations' -require 'knapsack' module QA module Specs @@ -17,44 +17,56 @@ module QA @options = [] end - def perform - args = [] - args.push('--tty') if tty + def paths_from_knapsack + allocator = Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator + + QA::Runtime::Logger.info '' + QA::Runtime::Logger.info 'Report specs:' + QA::Runtime::Logger.info allocator.report_node_tests.join(', ') + QA::Runtime::Logger.info '' + QA::Runtime::Logger.info 'Leftover specs:' + QA::Runtime::Logger.info allocator.leftover_node_tests.join(', ') + QA::Runtime::Logger.info '' + + ['--', allocator.node_tests] + end + + def rspec_tags + tags_for_rspec = [] if tags.any? - tags.each { |tag| args.push(['--tag', tag.to_s]) } + tags.each { |tag| tags_for_rspec.push(['--tag', tag.to_s]) } else - args.push(%w[--tag ~orchestrated]) unless (%w[-t --tag] & options).any? + tags_for_rspec.push(%w[--tag ~orchestrated]) unless (%w[-t --tag] & options).any? end - args.push(%w[--tag ~skip_signup_disabled]) if QA::Runtime::Env.signup_disabled? + tags_for_rspec.push(%w[--tag ~skip_signup_disabled]) if QA::Runtime::Env.signup_disabled? QA::Runtime::Env.supported_features.each_key do |key| - args.push(["--tag", "~requires_#{key}"]) unless QA::Runtime::Env.can_test? key + tags_for_rspec.push(%W[--tag ~requires_#{key}]) unless QA::Runtime::Env.can_test? key end - args.push(options) + tags_for_rspec + end - Runtime::Browser.configure! + def perform + args = [] + args.push('--tty') if tty + args.push(rspec_tags) + args.push(options) if Runtime::Env.knapsack? - allocator = Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator - - QA::Runtime::Logger.info '' - QA::Runtime::Logger.info 'Report specs:' - QA::Runtime::Logger.info allocator.report_node_tests.join(', ') - QA::Runtime::Logger.info '' - QA::Runtime::Logger.info 'Leftover specs:' - QA::Runtime::Logger.info allocator.leftover_node_tests.join(', ') - QA::Runtime::Logger.info '' - - args.push(['--', allocator.node_tests]) + args.push(paths_from_knapsack) else args.push(DEFAULT_TEST_PATH_ARGS) unless options.any? { |opt| opt =~ %r{/features/} } end - RSpec::Core::Runner.run(args.flatten, $stderr, $stdout).tap do |status| - abort if status.nonzero? + if Runtime::Scenario.attributes[:parallel] + ParallelRunner.run(args.flatten) + else + RSpec::Core::Runner.run(args.flatten, $stderr, $stdout).tap do |status| + abort if status.nonzero? + end end end end diff --git a/qa/qa/tools/generate_perf_testdata.rb b/qa/qa/tools/generate_perf_testdata.rb index b0477951967..26bcb2fe958 100644 --- a/qa/qa/tools/generate_perf_testdata.rb +++ b/qa/qa/tools/generate_perf_testdata.rb @@ -59,8 +59,8 @@ module QA group_search_response = create_a_group_api_req(@group_name, @visibility) group = JSON.parse(group_search_response.body) @urls[:group_page] = group["web_url"] - group["id"] STDOUT.puts "Created a group: #{@urls[:group_page]}" + group["id"] end def create_project(group_id) @@ -196,6 +196,7 @@ module QA project_path = "#{@group_name}%2F#{@project_name}" branch_name = "branch_with_many_commits-#{SecureRandom.hex(8)}" file_name = "file_for_many_commits.txt" + create_a_branch_api_req(branch_name, project_path) create_a_new_file_api_req(file_name, branch_name, project_path, "Initial commit for new file", "Initial file content") create_mr_response = create_a_merge_request_api_req(project_path, branch_name, "master", "MR with many commits-#{SecureRandom.hex(8)}") @@ -203,7 +204,7 @@ module QA 100.times do |i| update_file_api_req(file_name, branch_name, project_path, Faker::Lorem.sentences(5).join(" "), Faker::Lorem.sentences(500).join("\n")) end - STDOUT.puts "Created an MR with many commits: #{@urls[:mr_with_many_commits]}" + STDOUT.puts "Using branch: #{branch_name}, created an MR with many commits: #{@urls[:mr_with_many_commits]}" end private @@ -211,56 +212,88 @@ module QA # API Requests def create_a_discussion_on_issue_api_req(project_path_or_id, issue_id, body) - post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/issues/#{issue_id}/discussions").url, "body=\"#{body}\"" + call_api(expected_response_code: 201) do + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/issues/#{issue_id}/discussions").url, "body=\"#{body}\"" + end end def update_a_discussion_on_issue_api_req(project_path_or_id, mr_iid, discussion_id, resolved_status) - put Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/merge_requests/#{mr_iid}/discussions/#{discussion_id}").url, "resolved=#{resolved_status}" + call_api(expected_response_code: 200) do + put Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/merge_requests/#{mr_iid}/discussions/#{discussion_id}").url, "resolved=#{resolved_status}" + end end def create_a_discussion_on_mr_api_req(project_path_or_id, mr_iid, body) - post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/merge_requests/#{mr_iid}/discussions").url, - "body=\"#{body}\"" + call_api(expected_response_code: 201) do + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/merge_requests/#{mr_iid}/discussions").url, "body=\"#{body}\"" + end end def create_a_label_api_req(project_path_or_id, name, color) - post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/labels").url, "name=#{name}&color=#{color}" + call_api(expected_response_code: 201) do + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/labels").url, "name=#{name}&color=#{color}" + end end def create_a_todo_api_req(project_path_or_id, issue_id) - post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/issues/#{issue_id}/todo").url, nil + call_api(expected_response_code: 201) do + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/issues/#{issue_id}/todo").url, nil + end end def create_an_issue_api_req(project_path_or_id, title, description) - post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/issues").url, "title=#{title}&description=#{description}" + call_api(expected_response_code: 201) do + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/issues").url, "title=#{title}&description=#{description}" + end end def update_an_issue_api_req(project_path_or_id, issue_id, description, labels_list) - put Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/issues/#{issue_id}").url, "description=#{description}&labels=#{labels_list}" + call_api(expected_response_code: 200) do + put Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/issues/#{issue_id}").url, "description=#{description}&labels=#{labels_list}" + end end def create_a_project_api_req(project_name, group_id, visibility) - post Runtime::API::Request.new(@api_client, "/projects").url, "name=#{project_name}&namespace_id=#{group_id}&visibility=#{visibility}" + call_api(expected_response_code: 201) do + post Runtime::API::Request.new(@api_client, "/projects").url, "name=#{project_name}&namespace_id=#{group_id}&visibility=#{visibility}" + end end def create_a_group_api_req(group_name, visibility) - post Runtime::API::Request.new(@api_client, "/groups").url, "name=#{group_name}&path=#{group_name}&visibility=#{visibility}" + call_api(expected_response_code: 201) do + post Runtime::API::Request.new(@api_client, "/groups").url, "name=#{group_name}&path=#{group_name}&visibility=#{visibility}" + end end def create_a_branch_api_req(branch_name, project_path_or_id) - post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/repository/branches").url, "branch=#{branch_name}&ref=master" + call_api(expected_response_code: 201) do + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/repository/branches").url, "branch=#{branch_name}&ref=master" + end end def create_a_new_file_api_req(file_path, branch_name, project_path_or_id, commit_message, content) - post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/repository/files/#{file_path}").url, "branch=#{branch_name}&commit_message=\"#{commit_message}\"&content=\"#{content}\"" + call_api(expected_response_code: 201) do + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/repository/files/#{file_path}").url, "branch=#{branch_name}&commit_message=\"#{commit_message}\"&content=\"#{content}\"" + end end def create_a_merge_request_api_req(project_path_or_id, source_branch, target_branch, mr_title) - post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/merge_requests").url, "source_branch=#{source_branch}&target_branch=#{target_branch}&title=#{mr_title}" + call_api(expected_response_code: 201) do + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/merge_requests").url, "source_branch=#{source_branch}&target_branch=#{target_branch}&title=#{mr_title}" + end end def update_file_api_req(file_path, branch_name, project_path_or_id, commit_message, content) - put Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/repository/files/#{file_path}").url, "branch=#{branch_name}&commit_message=\"#{commit_message}\"&content=\"#{content}\"" + call_api(expected_response_code: 200) do + put Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/repository/files/#{file_path}").url, "branch=#{branch_name}&commit_message=\"#{commit_message}\"&content=\"#{content}\"" + end + end + + def call_api(expected_response_code: 200) + response = yield + raise "API call failed with response code: #{response.code} and body: #{response.body}" unless response.code == expected_response_code + + response end end end diff --git a/qa/spec/page/logging_spec.rb b/qa/spec/page/logging_spec.rb index 0f1ed039149..92a4f7b40e6 100644 --- a/qa/spec/page/logging_spec.rb +++ b/qa/spec/page/logging_spec.rb @@ -91,26 +91,26 @@ describe QA::Support::Page::Logging do it 'logs has_element?' do expect { subject.has_element?(:element) } - .to output(/has_element\? :element \(wait: 2\) returned: true/).to_stdout_from_any_process + .to output(/has_element\? :element \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/).to_stdout_from_any_process end it 'logs has_element? with text' do expect { subject.has_element?(:element, text: "some text") } - .to output(/has_element\? :element with text \"some text\" \(wait: 2\) returned: true/).to_stdout_from_any_process + .to output(/has_element\? :element with text \"some text\" \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/).to_stdout_from_any_process end it 'logs has_no_element?' do allow(page).to receive(:has_no_css?).and_return(true) expect { subject.has_no_element?(:element) } - .to output(/has_no_element\? :element \(wait: 2\) returned: true/).to_stdout_from_any_process + .to output(/has_no_element\? :element \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/).to_stdout_from_any_process end it 'logs has_no_element? with text' do allow(page).to receive(:has_no_css?).and_return(true) expect { subject.has_no_element?(:element, text: "more text") } - .to output(/has_no_element\? :element with text \"more text\" \(wait: 2\) returned: true/).to_stdout_from_any_process + .to output(/has_no_element\? :element with text \"more text\" \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/).to_stdout_from_any_process end it 'logs has_text?' do diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb index f25dbf3a8ab..21bfd2876a9 100644 --- a/qa/spec/spec_helper.rb +++ b/qa/spec/spec_helper.rb @@ -8,6 +8,10 @@ if ENV['CI'] && QA::Runtime::Env.knapsack? && !ENV['NO_KNAPSACK'] Knapsack::Adapters::RSpecAdapter.bind end +QA::Runtime::Browser.configure! + +QA::Runtime::Scenario.from_env(QA::Runtime::Env.runtime_scenario_attributes) if QA::Runtime::Env.runtime_scenario_attributes + %w[helpers shared_examples].each do |d| Dir[::File.join(__dir__, d, '**', '*.rb')].each { |f| require f } end diff --git a/qa/spec/specs/parallel_runner_spec.rb b/qa/spec/specs/parallel_runner_spec.rb new file mode 100644 index 00000000000..67d94a1f648 --- /dev/null +++ b/qa/spec/specs/parallel_runner_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +describe QA::Specs::ParallelRunner do + include Helpers::StubENV + + before do + allow(QA::Runtime::Scenario).to receive(:attributes).and_return(parallel: true) + stub_env('GITLAB_QA_ACCESS_TOKEN', 'skip_token_creation') + end + + it 'passes args to parallel_tests' do + expect_cli_arguments(['--tag', '~orchestrated', *QA::Specs::Runner::DEFAULT_TEST_PATH_ARGS]) + + subject.run(['--tag', '~orchestrated', *QA::Specs::Runner::DEFAULT_TEST_PATH_ARGS]) + end + + it 'passes a given test path to parallel_tests and adds a separator' do + expect_cli_arguments(%w[-- qa/specs/features/foo]) + + subject.run(%w[qa/specs/features/foo]) + end + + it 'passes tags and test paths to parallel_tests and adds a separator' do + expect_cli_arguments(%w[--tag smoke -- qa/specs/features/foo qa/specs/features/bar]) + + subject.run(%w[--tag smoke qa/specs/features/foo qa/specs/features/bar]) + end + + it 'passes tags and test paths with separators to parallel_tests' do + expect_cli_arguments(%w[-- --tag smoke -- qa/specs/features/foo qa/specs/features/bar]) + + subject.run(%w[-- --tag smoke -- qa/specs/features/foo qa/specs/features/bar]) + end + + it 'passes supported environment variables' do + # Test only env vars starting with GITLAB because some of the others + # affect how the runner behaves, and we're not concerned with those + # behaviors in this test + gitlab_env_vars = QA::Runtime::Env::ENV_VARIABLES.reject { |v| !v.start_with?('GITLAB') } + + gitlab_env_vars.each do |k, v| + stub_env(k, v) + end + + gitlab_env_vars['QA_RUNTIME_SCENARIO_ATTRIBUTES'] = '{"parallel":true}' + + expect_cli_arguments([], gitlab_env_vars) + + subject.run([]) + end + + def expect_cli_arguments(arguments, env = { 'QA_RUNTIME_SCENARIO_ATTRIBUTES' => '{"parallel":true}' }) + cmd = "bundle exec parallel_test -t rspec --combine-stderr --serialize-stdout -- #{arguments.join(' ')}" + expect(Open3).to receive(:popen2e) + .with(hash_including(env), cmd) + .and_return(0) + end +end diff --git a/qa/spec/specs/runner_spec.rb b/qa/spec/specs/runner_spec.rb index f94145d148e..6c533c6dc7d 100644 --- a/qa/spec/specs/runner_spec.rb +++ b/qa/spec/specs/runner_spec.rb @@ -58,11 +58,11 @@ describe QA::Specs::Runner do end end - context 'when "-- qa/specs/features/foo" is set as options' do - subject { described_class.new.tap { |runner| runner.options = %w[-- qa/specs/features/foo] } } + context 'when "--tag smoke" and "qa/specs/features/foo" are set as options' do + subject { described_class.new.tap { |runner| runner.options = %w[--tag smoke qa/specs/features/foo] } } - it 'passes the given tests path and excludes the orchestrated tag' do - expect_rspec_runner_arguments(['--tag', '~orchestrated', '--', 'qa/specs/features/foo']) + it 'focuses on the given tag and includes the path without excluding the orchestrated tag' do + expect_rspec_runner_arguments(['--tag', 'smoke', 'qa/specs/features/foo']) subject.perform end |