diff options
Diffstat (limited to 'qa')
126 files changed, 2509 insertions, 415 deletions
diff --git a/qa/.confiner/master.yml b/qa/.confiner/master.yml new file mode 100644 index 00000000000..bfb44facd7d --- /dev/null +++ b/qa/.confiner/master.yml @@ -0,0 +1,34 @@ +- name: Quarantine E2E tests in Master that fail consistently + plugin: + name: gitlab # https://gitlab.com/gitlab-org/quality/confiner/-/blob/main/doc/plugins/gitlab.md + args: + threshold: 3 # 3 failures + private_token: $QA_GITLAB_CI_TOKEN + project_id: gitlab-org/gitlab-qa-mirror # https://gitlab.com/gitlab-org/gitlab-qa-mirror/ + target_project: gitlab-org/gitlab + failure_issue_labels: QA,Quality + failure_issue_prefix: "Failure in " + pwd: qa # E2E specs reside in the qa subdirectory + timeout: 30 + ref: master + actions: + - quarantine + +- name: Dequarantine E2E tests in Master that pass consistently + plugin: + name: gitlab # https://gitlab.com/gitlab-org/quality/confiner/-/blob/main/doc/plugins/gitlab.md + args: + threshold: 10 # at least 10 passes consecutively with no failures to be a candidate for dequarantine + private_token: $QA_GITLAB_CI_TOKEN + + # we do not run quarantined jobs automatically on master, but we still commit to master + project_id: gitlab-org/quality/nightly # https://gitlab.com/gitlab-org/quality/nightly/ + target_project: gitlab-org/gitlab # https://gitlab.com/gitlab-org/gitlab + failure_issue_labels: QA,Quality + failure_issue_prefix: "Failure in " + pwd: qa # E2E specs reside in the qa subdirectory + timeout: 30 + ref: master + job_pattern: '.+-quarantine' + actions: + - dequarantine diff --git a/qa/.confiner/nightly.yml b/qa/.confiner/nightly.yml new file mode 100644 index 00000000000..78089525b0e --- /dev/null +++ b/qa/.confiner/nightly.yml @@ -0,0 +1,19 @@ +- name: Quarantine E2E tests in Nightly that fail consistently + plugin: + name: gitlab + args: + threshold: 3 + private_token: $QA_GITLAB_CI_TOKEN + project_id: gitlab-org/quality/nightly # https://gitlab.com/gitlab-org/quality/nightly/ + target_project: gitlab-org/gitlab + failure_issue_labels: QA,Quality,found:nightly + failure_issue_prefix: "Failure in " + pwd: qa + timeout: 30 + ref: master + environment: + name: nightly + pattern: 'pipeline: :nightly' + job_pattern: '^((?!quarantine).)*$' + actions: + - quarantine diff --git a/qa/.confiner/quarantine.yml b/qa/.confiner/quarantine.yml deleted file mode 100644 index 6534d72525d..00000000000 --- a/qa/.confiner/quarantine.yml +++ /dev/null @@ -1,15 +0,0 @@ -- name: Quarantine E2E tests that fail consistently - plugin: - name: gitlab # https://gitlab.com/gitlab-org/quality/confiner/-/blob/main/doc/plugins/gitlab.md - args: - threshold: 3 # 3 failures - private_token: $QA_GITLAB_CI_TOKEN - project_id: gitlab-org/gitlab-qa-mirror # https://gitlab.com/gitlab-org/gitlab-qa-mirror/ - target_project: gitlab-org/gitlab - failure_issue_labels: QA,Quality - failure_issue_prefix: "Failure in " - pwd: qa # E2E specs reside in the qa subdirectory - timeout: 30 - ref: master - actions: - - quarantine diff --git a/qa/.gitignore b/qa/.gitignore index b54b8666e28..3c5db4b565e 100644 --- a/qa/.gitignore +++ b/qa/.gitignore @@ -1,6 +1,8 @@ tmp/ +reports/ +no_of_examples/ + .ruby-version .tool-versions .ruby-gemset urls.yml -reports/ diff --git a/qa/.rspec_internal b/qa/.rspec_internal new file mode 100644 index 00000000000..ea32ca1e093 --- /dev/null +++ b/qa/.rspec_internal @@ -0,0 +1,4 @@ +--force-color +--order random +--format documentation +--require specs/spec_helper diff --git a/qa/Gemfile b/qa/Gemfile index 05acab5653f..b504d6d4e90 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -3,7 +3,7 @@ source 'https://rubygems.org' gem 'gitlab-qa', require: 'gitlab/qa' -gem 'activesupport', '~> 6.1.4.6' # This should stay in sync with the root's Gemfile +gem 'activesupport', '~> 6.1.4.7' # This should stay in sync with the root's Gemfile gem 'allure-rspec', '~> 2.16.0' gem 'capybara', '~> 3.35.0' gem 'capybara-screenshot', '~> 1.0.23' @@ -30,7 +30,7 @@ gem 'terminal-table', '~> 3.0.0', require: false gem 'slack-notifier', '~> 2.4', require: false gem 'fog-google', '~> 1.17', require: false -gem 'confiner', '~> 0.2' +gem 'confiner', '~> 0.3' gem 'chemlab', '~> 0.9' gem 'chemlab-library-www-gitlab-com', '~> 0.1' diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 4be8adaef33..c4809a17f66 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -57,8 +57,8 @@ GEM concord (0.1.5) adamantium (~> 0.2.0) equalizer (~> 0.0.9) - concurrent-ruby (1.1.9) - confiner (0.2.3) + concurrent-ruby (1.1.10) + confiner (0.3.0) gitlab (>= 4.17) zeitwerk (~> 2.5.1) declarative (0.0.20) @@ -357,14 +357,14 @@ PLATFORMS ruby DEPENDENCIES - activesupport (~> 6.1.4.6) + activesupport (~> 6.1.4.7) airborne (~> 0.3.4) allure-rspec (~> 2.16.0) capybara (~> 3.35.0) capybara-screenshot (~> 1.0.23) chemlab (~> 0.9) chemlab-library-www-gitlab-com (~> 0.1) - confiner (~> 0.2) + confiner (~> 0.3) deprecation_toolkit (~> 1.5.1) faker (~> 2.19, >= 2.19.0) fog-google (~> 1.17) diff --git a/qa/qa/fixtures/package_managers/npm/npm_upload_install_package_project.yaml.erb b/qa/qa/fixtures/package_managers/npm/npm_upload_install_package_project.yaml.erb index 8d94d03ef9b..5f2ff31b318 100644 --- a/qa/qa/fixtures/package_managers/npm/npm_upload_install_package_project.yaml.erb +++ b/qa/qa/fixtures/package_managers/npm/npm_upload_install_package_project.yaml.erb @@ -16,6 +16,7 @@ deploy: install: stage: install script: + - echo "//${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=<%= auth_token %>">.npmrc - "npm config set @<%= registry_scope %>:registry <%= gitlab_address_with_port %>/api/v4/projects/${CI_PROJECT_ID}/packages/npm/" - "npm install <%= package.name %>" cache: diff --git a/qa/qa/fixtures/script_extensions/test.html b/qa/qa/fixtures/script_extensions/test.html new file mode 100644 index 00000000000..0be2c080fd8 --- /dev/null +++ b/qa/qa/fixtures/script_extensions/test.html @@ -0,0 +1,6 @@ +<html> + <head></head> + <body> + <h1>Hello world</h1> + </body> +</html> diff --git a/qa/qa/flow/pipeline.rb b/qa/qa/flow/pipeline.rb index 999f352e834..d19b2530bb8 100644 --- a/qa/qa/flow/pipeline.rb +++ b/qa/qa/flow/pipeline.rb @@ -5,17 +5,24 @@ module QA module Pipeline module_function - # In some cases we don't need to wait for anything, blocked, running or pending is acceptable - # Some cases only we do need pipeline to finish with expected condition (completed, succeeded or replicated) - def visit_latest_pipeline(pipeline_condition: nil) + # Acceptable statuses: + # canceled, created, failed, manual, passed + # pending, running, skipped + def visit_latest_pipeline(status: nil, wait: nil, skip_wait: true) Page::Project::Menu.perform(&:click_ci_cd_pipelines) - Page::Project::Pipeline::Index.perform(&:"wait_for_latest_pipeline_#{pipeline_condition}") if pipeline_condition - Page::Project::Pipeline::Index.perform(&:click_on_latest_pipeline) + Page::Project::Pipeline::Index.perform do |index| + index.has_any_pipeline?(wait: wait) + index.wait_for_latest_pipeline(status: status, wait: wait) if status || !skip_wait + index.click_on_latest_pipeline + end end - def wait_for_latest_pipeline(pipeline_condition:) + def wait_for_latest_pipeline(status: nil, wait: nil) Page::Project::Menu.perform(&:click_ci_cd_pipelines) - Page::Project::Pipeline::Index.perform(&:"wait_for_latest_pipeline_#{pipeline_condition}") + Page::Project::Pipeline::Index.perform do |index| + index.has_any_pipeline?(wait: wait) + index.wait_for_latest_pipeline(status: status, wait: wait) + end end end end diff --git a/qa/qa/page/admin/settings/component/performance_bar.rb b/qa/qa/page/admin/settings/component/performance_bar.rb index 9e92fa362fb..ebf0e744b5e 100644 --- a/qa/qa/page/admin/settings/component/performance_bar.rb +++ b/qa/qa/page/admin/settings/component/performance_bar.rb @@ -12,7 +12,7 @@ module QA end def enable_performance_bar - check_element(:enable_performance_bar_checkbox) + check_element(:enable_performance_bar_checkbox, true) Capybara.current_session.driver.browser.manage.add_cookie(name: 'perf_bar_enabled', value: 'true') end diff --git a/qa/qa/page/admin/settings/component/snowplow.rb b/qa/qa/page/admin/settings/component/snowplow.rb index e05679feac3..c7f103e29a8 100644 --- a/qa/qa/page/admin/settings/component/snowplow.rb +++ b/qa/qa/page/admin/settings/component/snowplow.rb @@ -31,11 +31,11 @@ module QA private def check_snowplow_enabled_checkbox - check_element(:snowplow_enabled_checkbox) + check_element(:snowplow_enabled_checkbox, true) end def uncheck_snowplow_enabled_checkbox - uncheck_element(:snowplow_enabled_checkbox) + uncheck_element(:snowplow_enabled_checkbox, true) end def click_save_changes_button diff --git a/qa/qa/page/admin/settings/component/usage_statistics.rb b/qa/qa/page/admin/settings/component/usage_statistics.rb index 0275b7ae926..c296e63e28e 100644 --- a/qa/qa/page/admin/settings/component/usage_statistics.rb +++ b/qa/qa/page/admin/settings/component/usage_statistics.rb @@ -11,7 +11,7 @@ module QA end def has_disabled_usage_data_checkbox? - has_element?(:enable_usage_data_checkbox, disabled: true) + has_element?(:enable_usage_data_checkbox, disabled: true, visible: false) end end end diff --git a/qa/qa/page/base.rb b/qa/qa/page/base.rb index 526dd25ccc9..83db8bc0fd6 100644 --- a/qa/qa/page/base.rb +++ b/qa/qa/page/base.rb @@ -133,7 +133,9 @@ module QA end def all_elements(name, **kwargs) - if kwargs.keys.none? { |key| [:minimum, :maximum, :count, :between].include?(key) } + all_args = [:minimum, :maximum, :count, :between] + + if kwargs.keys.none? { |key| all_args.include?(key) } raise ArgumentError, "Please use :minimum, :maximum, :count, or :between so that all is more reliable" end @@ -247,6 +249,8 @@ module QA else find_element(name, **original_kwargs).disabled? == disabled end + rescue Capybara::ElementNotFound + false end # Check for the element before waiting for requests, just in case unrelated requests are in progress. @@ -467,8 +471,8 @@ module QA return element_when_flag_disabled if has_element?(element_when_flag_disabled) raise ElementNotFound, - "Could not find the expected element as #{element_when_flag_enabled} or #{element_when_flag_disabled}." \ - "The relevant feature flag is #{feature_flag}" + "Could not find the expected element as #{element_when_flag_enabled} or #{element_when_flag_disabled}." \ + "The relevant feature flag is #{feature_flag}" end end end diff --git a/qa/qa/page/component/access_tokens.rb b/qa/qa/page/component/access_tokens.rb index 3c8a6cf6a1d..6a9249621e1 100644 --- a/qa/qa/page/component/access_tokens.rb +++ b/qa/qa/page/component/access_tokens.rb @@ -19,7 +19,7 @@ module QA end base.view 'app/views/shared/tokens/_scopes_form.html.haml' do - element :api_checkbox, '#{scope}_checkbox' # rubocop:disable QA/ElementWithPattern, Lint/InterpolationCheck + element :api_label, '#{scope}_label' # rubocop:disable QA/ElementWithPattern, Lint/InterpolationCheck end base.view 'app/views/shared/access_tokens/_created_container.html.haml' do @@ -36,7 +36,7 @@ module QA end def check_api - check_element(:api_checkbox) + click_element(:api_label) end def click_create_token_button diff --git a/qa/qa/page/component/confirm_modal.rb b/qa/qa/page/component/confirm_modal.rb index a90be76c879..76200490f66 100644 --- a/qa/qa/page/component/confirm_modal.rb +++ b/qa/qa/page/component/confirm_modal.rb @@ -8,10 +8,14 @@ module QA def self.included(base) super + + base.view 'app/assets/javascripts/lib/utils/confirm_via_gl_modal/confirm_modal.vue' do + element :confirm_ok_button + end end def fill_confirmation_text(text) - fill_element :confirm_input, text + fill_element(:confirm_input, text) end def wait_for_confirm_button_enabled @@ -22,7 +26,11 @@ module QA def confirm_transfer wait_for_confirm_button_enabled - click_element :confirm_button + click_element(:confirm_button) + end + + def click_confirmation_ok_button + click_element(:confirm_ok_button) end end end diff --git a/qa/qa/page/component/members_filter.rb b/qa/qa/page/component/members_filter.rb index ac07fe7e9fa..fce4560d255 100644 --- a/qa/qa/page/component/members_filter.rb +++ b/qa/qa/page/component/members_filter.rb @@ -10,15 +10,14 @@ module QA super base.view 'app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue' do - element :members_filtered_search_bar_content + element :search_bar_input + element :search_button end end def search_member(username) - # TODO: Update the two actions below to use direct qa selectors once this is implemented: - # https://gitlab.com/gitlab-org/gitlab-ui/-/issues/1688 - find_element(:members_filtered_search_bar_content).find('input').set(username) - find('.gl-search-box-by-click-search-button').click + fill_element :search_bar_input, username + click_element :search_button end end end diff --git a/qa/qa/page/component/wiki_page_form.rb b/qa/qa/page/component/wiki_page_form.rb index 8f504b784b2..74b6c6b2d5e 100644 --- a/qa/qa/page/component/wiki_page_form.rb +++ b/qa/qa/page/component/wiki_page_form.rb @@ -14,7 +14,6 @@ module QA element :wiki_content_textarea element :wiki_message_textbox element :wiki_submit_button - element :try_new_editor_container element :editing_mode_button end diff --git a/qa/qa/page/dashboard/todos.rb b/qa/qa/page/dashboard/todos.rb index d8baadcf73d..d5660823118 100644 --- a/qa/qa/page/dashboard/todos.rb +++ b/qa/qa/page/dashboard/todos.rb @@ -17,7 +17,11 @@ module QA end def has_todo_list? - has_element? :todo_item_container + has_element?(:todo_item_container) + end + + def has_no_todo_list? + has_no_element?(:todo_item_container) end def has_latest_todo_item_with_content?(action, title) diff --git a/qa/qa/page/main/login.rb b/qa/qa/page/main/login.rb index c34b8f33a5d..d8b7bb90437 100644 --- a/qa/qa/page/main/login.rb +++ b/qa/qa/page/main/login.rb @@ -130,6 +130,10 @@ module QA has_css?(".active", text: 'Standard') end + def has_arkose_labs_token? + has_css?('[name="arkose_labs_token"][value]', visible: false) + end + def switch_to_sign_in_tab click_element :sign_in_tab end @@ -174,6 +178,17 @@ module QA fill_element :login_field, user.username fill_element :password_field, user.password + + if Runtime::Env.running_on_dot_com? + # Arkose only appears in staging.gitlab.com, gitlab.com, etc... + + # Wait until the ArkoseLabs challenge has initialized + Support::WaitForRequests.wait_for_requests + Support::Waiter.wait_until(max_duration: 5, reload_page: false, raise_on_failure: false) do + has_arkose_labs_token? + end + end + click_element :sign_in_button Support::WaitForRequests.wait_for_requests diff --git a/qa/qa/page/main/menu.rb b/qa/qa/page/main/menu.rb index e3bb585955b..0bb74455f81 100644 --- a/qa/qa/page/main/menu.rb +++ b/qa/qa/page/main/menu.rb @@ -45,6 +45,14 @@ module QA element :search_term_field end + view 'app/views/layouts/_header_search.html.haml' do + element :search_box + end + + view 'app/assets/javascripts/header_search/components/app.vue' do + element :search_term_field + end + def go_to_groups within_groups_menu do click_element(:menu_item_link, title: 'Your groups') @@ -146,6 +154,7 @@ module QA end def search_for(term) + click_element(:search_box) fill_element :search_term_field, "#{term}\n" end diff --git a/qa/qa/page/project/import/github.rb b/qa/qa/page/project/import/github.rb index 47f7e701ae8..89d044bac8d 100644 --- a/qa/qa/page/project/import/github.rb +++ b/qa/qa/page/project/import/github.rb @@ -75,11 +75,10 @@ module QA # @return [Boolean] def has_imported_project?(gh_project_name, wait: QA::Support::WaitForRequests::DEFAULT_MAX_WAIT_TIME) within_element(:project_import_row, source_project: gh_project_name, skip_finished_loading_check: true) do - # TODO: remove retrier with reload:true once https://gitlab.com/gitlab-org/gitlab/-/issues/292861 is fixed wait_until( max_duration: wait, sleep_interval: 5, - reload: true, + reload: false, skip_finished_loading_check_on_refresh: true ) do has_element?(:import_status_indicator, text: "Complete") diff --git a/qa/qa/page/project/infrastructure/kubernetes/index.rb b/qa/qa/page/project/infrastructure/kubernetes/index.rb index 0424682179e..34d2ad55429 100644 --- a/qa/qa/page/project/infrastructure/kubernetes/index.rb +++ b/qa/qa/page/project/infrastructure/kubernetes/index.rb @@ -6,12 +6,13 @@ module QA module Infrastructure module Kubernetes class Index < Page::Base - view 'app/assets/javascripts/clusters_list/components/clusters_view_all.vue' do - element :connect_existing_cluster_button + view 'app/assets/javascripts/clusters_list/components/clusters_actions.vue' do + element :clusters_actions_button end def connect_existing_cluster - click_link 'Connect existing cluster' + within_element(:clusters_actions_button) { click_button(class: 'dropdown-toggle-split') } + click_link 'Connect a cluster (certificate - deprecated)' end def has_cluster?(cluster) diff --git a/qa/qa/page/project/new.rb b/qa/qa/page/project/new.rb index 340e40127c9..26fff85dd99 100644 --- a/qa/qa/page/project/new.rb +++ b/qa/qa/page/project/new.rb @@ -13,6 +13,7 @@ module QA view 'app/views/projects/_new_project_fields.html.haml' do element :initialize_with_readme_checkbox + element :initialize_with_sast_checkbox element :project_name element :project_path element :project_description @@ -20,10 +21,6 @@ module QA element :visibility_radios end - view 'app/views/projects/_new_project_initialize_with_sast.html.haml' do - element :initialize_with_sast_checkbox - end - view 'app/views/projects/project_templates/_template.html.haml' do element :use_template_button element :template_option_row diff --git a/qa/qa/page/project/pipeline/index.rb b/qa/qa/page/project/pipeline/index.rb index d088ba76bc0..70ac35eeffe 100644 --- a/qa/qa/page/project/pipeline/index.rb +++ b/qa/qa/page/project/pipeline/index.rb @@ -21,54 +21,46 @@ module QA element :run_pipeline_button end - def click_on_latest_pipeline - all_elements(:pipeline_url_link, minimum: 1, wait: QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME).first.click - end - - def wait_for_latest_pipeline_succeeded - wait_for_latest_pipeline_status { has_selector?(".ci-status-icon-success") } + view 'app/assets/javascripts/pipelines/components/pipelines_list/pipelines_table.vue' do + element :pipeline_row_container end - def wait_for_latest_pipeline_completed - wait_for_latest_pipeline_status { has_selector?(".ci-status-icon-success") || has_selector?(".ci-status-icon-failed") } + def latest_pipeline + all_elements(:pipeline_row_container, minimum: 1).first end - def wait_for_latest_pipeline_skipped - wait_for_latest_pipeline_status { has_text?('skipped') } + def latest_pipeline_status + latest_pipeline.find(element_selector_css(:pipeline_commit_status)).text end - def wait_for_latest_pipeline_status - wait_until(max_duration: 90, reload: true, sleep_interval: 5) { has_pipeline? } + # If no status provided, wait for pipeline to complete + def wait_for_latest_pipeline(status: nil, wait: nil, reload: false) + wait ||= Support::Repeater::DEFAULT_MAX_WAIT_TIME + finished_status = %w[passed failed canceled skipped manual] - wait_until(reload: false, max_duration: 360) do - within_element_by_index(:pipeline_commit_status, 0) { yield } + wait_until(max_duration: wait, reload: reload, sleep_interval: 1) do + status ? latest_pipeline_status == status : finished_status.include?(latest_pipeline_status) end end - def wait_for_latest_pipeline_success_or_retry - wait_for_latest_pipeline_completion - - if has_text?('failed') - click_element :pipeline_retry_button - wait_for_latest_pipeline_success - end - end - - def has_pipeline? - has_element? :pipeline_url_link + def has_any_pipeline?(wait: nil) + wait ||= Support::Repeater::DEFAULT_MAX_WAIT_TIME + wait_until(max_duration: wait) { has_element?(:pipeline_row_container) } end def has_no_pipeline? - has_no_element? :pipeline_url_link + has_no_element?(:pipeline_row_container) end def click_run_pipeline_button - click_element :run_pipeline_button, Page::Project::Pipeline::New + click_element(:run_pipeline_button, Page::Project::Pipeline::New) + end + + def click_on_latest_pipeline + latest_pipeline.find(element_selector_css(:pipeline_url_link)).click end end end end end end - -QA::Page::Project::Pipeline::Index.prepend_mod_with('Page::Project::Pipeline::Index', namespace: QA) diff --git a/qa/qa/page/project/pipeline/new.rb b/qa/qa/page/project/pipeline/new.rb index 644a21b46e9..96a48e6240a 100644 --- a/qa/qa/page/project/pipeline/new.rb +++ b/qa/qa/page/project/pipeline/new.rb @@ -7,10 +7,20 @@ module QA class New < QA::Page::Base view 'app/assets/javascripts/pipeline_new/components/pipeline_new_form.vue' do element :run_pipeline_button, required: true + element :ci_variable_row_container + element :ci_variable_key_field + element :ci_variable_value_field end def click_run_pipeline_button - click_element :run_pipeline_button + click_element(:run_pipeline_button, Page::Project::Pipeline::Show) + end + + def add_variable(key, value, row_index: 0) + within_element_by_index(:ci_variable_row_container, row_index) do + fill_element(:ci_variable_key_field, key) + fill_element(:ci_variable_value_field, value) + end end end end diff --git a/qa/qa/page/project/pipeline/show.rb b/qa/qa/page/project/pipeline/show.rb index 6f4757a34e8..f499b748fb4 100644 --- a/qa/qa/page/project/pipeline/show.rb +++ b/qa/qa/page/project/pipeline/show.rb @@ -8,7 +8,7 @@ module QA include Component::CiBadgeLink view 'app/assets/javascripts/vue_shared/components/header_ci_component.vue' do - element :pipeline_header + element :pipeline_header, required: true end view 'app/assets/javascripts/pipelines/components/graph/graph_component.vue' do @@ -16,14 +16,14 @@ module QA end view 'app/assets/javascripts/pipelines/components/graph/job_item.vue' do - element :job_item_container - element :job_link + element :job_item_container, required: true + element :job_link, required: true element :job_action_button end view 'app/assets/javascripts/pipelines/components/graph/linked_pipeline.vue' do - element :expand_pipeline_button - element :child_pipeline + element :expand_linked_pipeline_button + element :linked_pipeline_container end view 'app/assets/javascripts/reports/components/report_section.vue' do @@ -73,14 +73,18 @@ module QA end end - def has_child_pipeline?(title: nil) - title ? find_child_pipeline_by_title(title) : has_element?(:child_pipeline) + def has_linked_pipeline?(title: nil) + title ? find_linked_pipeline_by_title(title) : has_element?(:linked_pipeline_container) end - def has_no_child_pipeline? - has_no_element?(:child_pipeline) + alias_method :has_child_pipeline?, :has_linked_pipeline? + + def has_no_linked_pipeline? + has_no_element?(:linked_pipeline_container) end + alias_method :has_no_child_pipeline?, :has_no_linked_pipeline? + def click_job(job_name) # Retry due to transient bug https://gitlab.com/gitlab-org/gitlab/-/issues/347126 QA::Support::Retrier.retry_on_exception do @@ -88,22 +92,24 @@ module QA end end - def child_pipelines - all_elements(:child_pipeline, minimum: 1) + def linked_pipelines + all_elements(:linked_pipeline_container, minimum: 1) end - def find_child_pipeline_by_title(title) - child_pipelines.find { |pipeline| pipeline[:title].include?(title) } + def find_linked_pipeline_by_title(title) + linked_pipelines.find { |pipeline| pipeline[:title].include?(title) } end - def expand_child_pipeline(title: nil) - child_pipeline = title ? find_child_pipeline_by_title(title) : child_pipelines.first + def expand_linked_pipeline(title: nil) + linked_pipeline = title ? find_linked_pipeline_by_title(title) : linked_pipelines.first - within_element_by_index(:child_pipeline, child_pipelines.index(child_pipeline)) do - click_element(:expand_pipeline_button) + within_element_by_index(:linked_pipeline_container, linked_pipelines.index(linked_pipeline)) do + click_element(:expand_linked_pipeline_button) end end + alias_method :expand_child_pipeline, :expand_linked_pipeline + def expand_license_report within_element(:license_report_widget) do click_element(:expand_report_button) diff --git a/qa/qa/page/project/pipeline_editor/show.rb b/qa/qa/page/project/pipeline_editor/show.rb index caf54a10025..1a8e1e07994 100644 --- a/qa/qa/page/project/pipeline_editor/show.rb +++ b/qa/qa/page/project/pipeline_editor/show.rb @@ -15,9 +15,9 @@ module QA element :target_branch_field, required: true end - view 'app/assets/javascripts/pipeline_editor/components/drawer/pipeline_editor_drawer.vue' do - element :toggle_sidebar_collapse_button - element :drawer_content + view 'app/assets/javascripts/pipeline_editor/components/editor/ci_editor_header.vue' do + element :drawer_toggle, required: true + element :template_repo_link, required: true end view 'app/assets/javascripts/vue_shared/components/source_editor.vue' do @@ -46,13 +46,6 @@ module QA element :file_editor_container end - def initialize - super - - wait_for_requests - close_toggle_sidebar - end - def open_branch_selector_dropdown click_element(:branch_selector_button) end @@ -148,15 +141,6 @@ module QA find('.nav-item', text: name).click end end - - # If the page thinks user has never opened pipeline editor before - # It will expand pipeline editor sidebar by default - # Collapse the sidebar if it is expanded - def close_toggle_sidebar - return unless has_element?(:drawer_content) - - click_element(:toggle_sidebar_collapse_button) - end end end end diff --git a/qa/qa/page/project/settings/services/jenkins.rb b/qa/qa/page/project/settings/services/jenkins.rb index 3d7da8d0161..8e092371491 100644 --- a/qa/qa/page/project/settings/services/jenkins.rb +++ b/qa/qa/page/project/settings/services/jenkins.rb @@ -7,10 +7,10 @@ module QA module Services class Jenkins < QA::Page::Base view 'app/assets/javascripts/integrations/edit/components/dynamic_field.vue' do - element :jenkins_url_field, ':data-qa-selector="`${fieldId}_field`"' # rubocop:disable QA/ElementWithPattern - element :project_name_field, ':data-qa-selector="`${fieldId}_field`"' # rubocop:disable QA/ElementWithPattern - element :username_field, ':data-qa-selector="`${fieldId}_field`"' # rubocop:disable QA/ElementWithPattern - element :password_field, ':data-qa-selector="`${fieldId}_field`"' # rubocop:disable QA/ElementWithPattern + element :service_jenkins_url_field, ':data-qa-selector="`${fieldId}_field`"' # rubocop:disable QA/ElementWithPattern + element :service_project_name_field, ':data-qa-selector="`${fieldId}_field`"' # rubocop:disable QA/ElementWithPattern + element :service_username_field, ':data-qa-selector="`${fieldId}_field`"' # rubocop:disable QA/ElementWithPattern + element :service_password_field, ':data-qa-selector="`${fieldId}_field`"' # rubocop:disable QA/ElementWithPattern end view 'app/assets/javascripts/integrations/edit/components/integration_form.vue' do @@ -28,19 +28,19 @@ module QA private def set_jenkins_url(jenkins_url) - fill_element(:jenkins_url_field, jenkins_url) + fill_element(:service_jenkins_url_field, jenkins_url) end def set_project_name(project_name) - fill_element(:project_name_field, project_name) + fill_element(:service_project_name_field, project_name) end def set_username(username) - fill_element(:username_field, username) + fill_element(:service_username_field, username) end def set_password(password) - fill_element(:password_field, password) + fill_element(:service_password_field, password) end def click_save_changes_button diff --git a/qa/qa/resource/bulk_import_group.rb b/qa/qa/resource/bulk_import_group.rb index a22529152e1..31db8ae4cc6 100644 --- a/qa/qa/resource/bulk_import_group.rb +++ b/qa/qa/resource/bulk_import_group.rb @@ -7,10 +7,14 @@ module QA :destination_group, :import_id - attribute :access_token do + attribute :import_access_token do api_client.personal_access_token end + attribute :gitlab_address do + QA::Runtime::Scenario.gitlab_address + end + # In most cases we will want to set path the same as source group # but it can be set to a custom name as well when imported via API attribute :destination_group_path do @@ -19,18 +23,16 @@ module QA # Can't define path as attribue since @path is set in base class initializer alias_method :path, :destination_group_path - delegate :gitlab_address, to: 'QA::Runtime::Scenario' - - def fabricate_via_browser_ui! + def fabricate! Page::Main::Menu.perform(&:go_to_create_group) Page::Group::New.perform do |group| group.switch_to_import_tab - group.connect_gitlab_instance(gitlab_address, api_client.personal_access_token) + group.connect_gitlab_instance(gitlab_address, import_access_token) end Page::Group::BulkImport.perform do |import_page| - import_page.import_group(path, sandbox.path) + import_page.import_group(destination_group_path, sandbox.full_path) end reload! @@ -49,7 +51,7 @@ module QA { configuration: { url: gitlab_address, - access_token: access_token + access_token: import_access_token }, entities: [ { diff --git a/qa/qa/resource/project.rb b/qa/qa/resource/project.rb index 740a8920cf2..dba3eb2e219 100644 --- a/qa/qa/resource/project.rb +++ b/qa/qa/resource/project.rb @@ -90,7 +90,12 @@ module QA Page::Project::New.perform(&:click_blank_project_link) Page::Project::New.perform do |new_page| - new_page.choose_test_namespace unless @personal_namespace + if @personal_namespace + new_page.choose_namespace(@personal_namespace) + else + new_page.choose_test_namespace + end + new_page.choose_name(@name) new_page.add_description(@description) new_page.set_visibility(@visibility) @@ -105,7 +110,16 @@ module QA def fabricate_via_api! resource_web_url(api_get) rescue ResourceNotFoundError - super + response = super + + # If a project is being imported, wait until it completes before we let the test continue. + # Otherwise we see Git repository errors + # See https://gitlab.com/gitlab-org/gitlab/-/issues/356101 + Support::Retrier.retry_until(max_duration: 60, sleep_interval: 5) do + %w[none finished].include?(reload!.api_resource[:import_status]) + end + + response end def api_get_path @@ -295,13 +309,6 @@ module QA merge_requests.find { |mr| mr[:title] == title } end - def runners(tag_list: nil) - url = tag_list ? "#{api_runners_path}?tag_list=#{tag_list.compact.join(',')}" : api_runners_path - response = get(request_url(url, per_page: '100')) - - parse_body(response) - end - def registry_repositories response = get(request_url(api_registry_repositories_path)) parse_body(response) @@ -336,16 +343,17 @@ module QA parse_body(response) end - def pipelines - response = get(request_url(api_pipelines_path)) - parse_body(response) - end - def pipeline_schedules response = get(request_url(api_pipeline_schedules_path)) parse_body(response) end + def pipelines(auto_paginate: false, attempts: 0) + return parse_body(api_get_from(api_pipelines_path)) unless auto_paginate + + auto_paginated_response(request_url(api_pipelines_path, per_page: '100'), attempts: attempts) + end + def issues(auto_paginate: false, attempts: 0) return parse_body(api_get_from(api_issues_path)) unless auto_paginate @@ -380,9 +388,7 @@ module QA api_resource[:import_status] == "finished" end - unless mirror_succeeded - raise "Mirroring failed with error: #{api_resource[:import_error]}" - end + raise "Mirroring failed with error: #{api_resource[:import_error]}" unless mirror_succeeded end def remove_via_api! diff --git a/qa/qa/resource/runner.rb b/qa/qa/resource/runner.rb index 9c5c9992442..c014563671d 100644 --- a/qa/qa/resource/runner.rb +++ b/qa/qa/resource/runner.rb @@ -31,7 +31,7 @@ module QA end def fabricate_via_api! - Service::DockerRun::GitlabRunner.new(name).tap do |runner| + @docker_container = Service::DockerRun::GitlabRunner.new(name).tap do |runner| runner.pull runner.token = @token ||= project.runners_token runner.address = Runtime::Scenario.gitlab_address @@ -46,12 +46,22 @@ module QA end def remove_via_api! - runners = project.runners(tag_list: @tags) + runners = list_of_runners(tag_list: @tags) - return if runners.blank? + # If we have no runners, print the logs from the runner docker container in case they show why it isn't running. + if runners.blank? + dump_logs + + return + end this_runner = runners.find { |runner| runner[:description] == name } + + # As above, but now we should have a specific runner. If not, print the logs from the runner docker container + # to see if we can find out why the runner isn't running. unless this_runner + dump_logs + raise "Project #{project.path_with_namespace} does not have a runner with a description matching #{name} #{"or tags #{@tags}" if @tags&.any?}. Runners available: #{runners}" end @@ -62,6 +72,20 @@ module QA Service::DockerRun::GitlabRunner.new(name).remove! end + def list_of_runners(tag_list: nil) + url = tag_list ? "#{api_post_path}?tag_list=#{tag_list.compact.join(',')}" : api_post_path + response = get(request_url(url, per_page: '100')) + + # Capturing 500 error code responses to log this issue better. We can consider cleaning it up once https://gitlab.com/gitlab-org/gitlab/-/issues/331753 is addressed. + raise "Response returned a #{response.code} error code. #{response.body}" if response.code == Support::API::HTTP_STATUS_SERVER_ERROR + + parse_body(response) + end + + def reload! + super if method(:running?).super_method.call + end + def api_delete_path "/runners/#{id}" end @@ -70,10 +94,21 @@ module QA end def api_post_path + "/runners" end def api_post_body end + + private + + def dump_logs + if @docker_container.running? + @docker_container.logs { |line| QA::Runtime::Logger.debug(line) } + else + QA::Runtime::Logger.debug("No runner container found named #{name}") + end + end end end end diff --git a/qa/qa/runtime/api/repository_storage_moves.rb b/qa/qa/runtime/api/repository_storage_moves.rb index c3b2095be32..fb8d70c0836 100644 --- a/qa/qa/runtime/api/repository_storage_moves.rb +++ b/qa/qa/runtime/api/repository_storage_moves.rb @@ -24,7 +24,7 @@ module QA Logger.debug('Getting repository storage moves') Support::Waiter.wait_until do - with_paginated_response_body(Request.new(api_client, "/#{resource_name(resource)}_repository_storage_moves", per_page: '100').url) do |page| + get(Request.new(api_client, "/#{resource_name(resource)}_repository_storage_moves", per_page: '100').url) do |page| break true if page.any? { |item| yield item } end end diff --git a/qa/qa/runtime/browser.rb b/qa/qa/runtime/browser.rb index f1d93ce376a..89e84f414b1 100644 --- a/qa/qa/runtime/browser.rb +++ b/qa/qa/runtime/browser.rb @@ -194,6 +194,8 @@ module QA def initialize(instance, page_class) @session_address = Runtime::Address.new(instance, page_class) @page_class = page_class + + Session.enable_interception if Runtime::Env.can_intercept? end def url @@ -255,6 +257,27 @@ module QA @network_conditions_configured = false end + def self.enable_interception + script = File.read("#{__dir__}/script_extensions/interceptor.js") + command = { + cmd: 'Page.addScriptToEvaluateOnNewDocument', + params: { + source: script + } + } + @interceptor_script_params = Capybara.current_session.driver.browser.send(:bridge).send_command(command) + end + + def self.disable_interception + return unless @interceptor_script_params + + command = { + cmd: 'Page.removeScriptToEvaluateOnNewDocument', + params: @interceptor_script_params + } + Capybara.current_session.driver.browser.send(:bridge).send_command(command) + end + private def simulate_slow_connection diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb index 63207751c78..e4537009406 100644 --- a/qa/qa/runtime/env.rb +++ b/qa/qa/runtime/env.rb @@ -37,6 +37,14 @@ module QA ENV['QA_PRAEFECT_REPOSITORY_STORAGE'] end + def interception_enabled? + enabled?(ENV['INTERCEPT_REQUESTS'], default: true) + end + + def can_intercept? + browser == :chrome && interception_enabled? + end + def ci_job_url ENV['CI_JOB_URL'] end diff --git a/qa/qa/runtime/script_extensions/interceptor.js b/qa/qa/runtime/script_extensions/interceptor.js new file mode 100644 index 00000000000..9e98b0421b4 --- /dev/null +++ b/qa/qa/runtime/script_extensions/interceptor.js @@ -0,0 +1,158 @@ +(() => { + const CACHE_NAME = 'INTERCEPTOR_CACHE'; + + /** + * Fetches and parses JSON from the sessionStorage cache + * @returns {(Object)} + */ + const getCache = () => { + return JSON.parse(sessionStorage.getItem(CACHE_NAME)); + }; + + /** + * Commits an object to the sessionStorage cache + * @param {Object} data + */ + const saveCache = (data) => { + sessionStorage.setItem(CACHE_NAME, JSON.stringify(data)); + }; + + /** + * Checks if the cache is available + * and if the current context has access to it + * @returns {boolean} can we access the cache? + */ + const checkCache = () => { + try { + getCache(); + return true; + } catch (error) { + // eslint-disable-next-line no-console + console.warn(`Couldn't access cache: ${error.toString()}`); + return false; + } + }; + + /** + * @callback cacheCommitCallback + * @param {object} cache + * @return {object} mutated cache + */ + + /** + * If the cache is available, takes a callback function that is called + * with an object returned from getCache, + * and saves whatever is returned from the callback function + * to the cache + * @param {cacheCommitCallback} cb + */ + const commitToCache = (cb) => { + if (checkCache()) { + const cache = cb(getCache()); + saveCache(cache); + } + }; + + window.Interceptor = { + saveCache, + commitToCache, + getCache, + checkCache, + activeFetchRequests: 0, + }; + + const pureFetch = window.fetch; + const pureXHROpen = window.XMLHttpRequest.prototype.open; + + /** + * Replacement for XMLHttpRequest.prototype.open + * listens for complete xhr events + * if the xhr response has a status code higher than 400 + * then commit request/response metadata to the cache + * @param method intercepted HTTP method (GET|POST|etc..) + * @param url intercepted HTTP url + * @param args intercepted XHR arguments (credentials, headers, options + * @return {Promise} the result of the original XMLHttpRequest.prototype.open implementation + */ + function interceptXhr(method, url, ...args) { + this.addEventListener( + 'readystatechange', + () => { + const self = this; + if (this.readyState === XMLHttpRequest.DONE) { + if (this.status >= 400 || this.status === 0) { + commitToCache((cache) => { + // eslint-disable-next-line no-param-reassign + cache.errors ||= []; + cache.errors.push({ + status: self.status === 0 ? -1 : self.status, + url, + method, + headers: { 'x-request-id': self.getResponseHeader('x-request-id') }, + }); + return cache; + }); + } + } + }, + false, + ); + return pureXHROpen.apply(this, [method, url, ...args]); + } + + /** + * Replacement for fetch implementation + * tracks active requests, and commits metadata to the cache + * if the response is not ok or was cancelled. + * Additionally tracks activeFetchRequests on the Interceptor object + * @param url target HTTP url + * @param opts fetch options, including request method, body, etc + * @param args additional fetch arguments + * @returns {Promise<"success"|"error">} the result of the original fetch call + */ + async function interceptedFetch(url, opts, ...args) { + const method = opts && opts.method ? opts.method : 'GET'; + window.Interceptor.activeFetchRequests += 1; + try { + const response = await pureFetch(url, opts, ...args); + window.Interceptor.activeFetchRequests += -1; + const clone = response.clone(); + + if (!clone.ok) { + commitToCache((cache) => { + // eslint-disable-next-line no-param-reassign + cache.errors ||= []; + cache.errors.push({ + status: clone.status, + url, + method, + headers: { 'x-request-id': clone.headers.get('x-request-id') }, + }); + return cache; + }); + } + return response; + } catch (error) { + commitToCache((cache) => { + // eslint-disable-next-line no-param-reassign + cache.errors ||= []; + cache.errors.push({ + status: -1, + url, + method, + }); + return cache; + }); + + window.Interceptor.activeFetchRequests += -1; + throw error; + } + } + + if (checkCache()) { + saveCache({}); + } + + window.fetch = interceptedFetch; + window.XMLHttpRequest.prototype.open = interceptXhr; +})(); diff --git a/qa/qa/scenario/bootable.rb b/qa/qa/scenario/bootable.rb index 2a9bbbc9fdb..8eedfab3cff 100644 --- a/qa/qa/scenario/bootable.rb +++ b/qa/qa/scenario/bootable.rb @@ -31,7 +31,7 @@ module QA end next - elsif opt.name == :count_examples_only + elsif opt.name == :count_examples_only || opt.name == :test_metadata_only parser.on(opt.arg, opt.desc) do |value| QA::Runtime::Env.dry_run = true Runtime::Scenario.define(opt.name, value) diff --git a/qa/qa/scenario/shared_attributes.rb b/qa/qa/scenario/shared_attributes.rb index d5d7aedb47f..ad2d2d3a960 100644 --- a/qa/qa/scenario/shared_attributes.rb +++ b/qa/qa/scenario/shared_attributes.rb @@ -14,6 +14,7 @@ module QA attribute :parallel, '--parallel', 'Execute tests in parallel' attribute :loop, '--loop', 'Execute test repeatedly' attribute :count_examples_only, '--count-examples-only', 'Return the number of examples without running them' + attribute :test_metadata_only, '--test-metadata-only', 'Return all e2e test metadata without running them' end end end diff --git a/qa/qa/service/docker_run/base.rb b/qa/qa/service/docker_run/base.rb index 512960e8232..85c06e6c307 100644 --- a/qa/qa/service/docker_run/base.rb +++ b/qa/qa/service/docker_run/base.rb @@ -11,6 +11,12 @@ module QA @runner_network = Runtime::Scenario.attributes[:runner_network] || @network end + def logs + shell "docker logs #{@name}" do |line| + yield " #{line.chomp}" + end + end + def network shell "docker network inspect #{@network}" rescue CommandError diff --git a/qa/qa/service/docker_run/gitlab_runner.rb b/qa/qa/service/docker_run/gitlab_runner.rb index 595d47bf162..0a8ac39dabd 100644 --- a/qa/qa/service/docker_run/gitlab_runner.rb +++ b/qa/qa/service/docker_run/gitlab_runner.rb @@ -43,6 +43,8 @@ module QA #{@image} #{add_gitlab_tls_cert if @address.include? "https"} && docker exec --detach #{@name} sh -c "#{register_command}" CMD + wait_until_running_and_configured + # Prove airgappedness if runner_network == 'airgapped' shell("docker exec #{@name} sh -c '#{prove_airgap}'") @@ -111,6 +113,10 @@ module QA && docker cp #{gitlab_tls_certificate.path} #{@name}:/etc/gitlab-runner/certs/gitlab.test.crt CMD end + + def wait_until_running_and_configured + wait_until_shell_command_matches("docker logs #{@name}", /Configuration loaded/) + end end end end diff --git a/qa/qa/service/praefect_manager.rb b/qa/qa/service/praefect_manager.rb index c364b00629c..1215268919c 100644 --- a/qa/qa/service/praefect_manager.rb +++ b/qa/qa/service/praefect_manager.rb @@ -530,9 +530,17 @@ module QA storage_repositories[2..-3] end + def modify_repo_access_time(node, repo_path, update_time) + repo = "/var/opt/gitlab/git-data/repositories/#{repo_path}" + shell(%{ + docker exec --user git #{node} bash -c 'find #{repo} -exec touch -d "#{update_time}" {} \\;' + }) + end + def add_repo_to_disk(node, repo_path) cmd = "GIT_DIR=. git init --initial-branch=main /var/opt/gitlab/git-data/repositories/#{repo_path}" shell "docker exec --user git #{node} bash -c '#{cmd}'" + modify_repo_access_time(node, repo_path, "24 hours ago") end def remove_repo_from_disk(repo_path) diff --git a/qa/qa/specs/features/api/1_manage/import_github_repo_spec.rb b/qa/qa/specs/features/api/1_manage/import_github_repo_spec.rb index c39db63f64d..79bba484bea 100644 --- a/qa/qa/specs/features/api/1_manage/import_github_repo_spec.rb +++ b/qa/qa/specs/features/api/1_manage/import_github_repo_spec.rb @@ -1,7 +1,9 @@ # frozen_string_literal: true module QA - RSpec.describe 'Manage', :github, :requires_admin, :reliable do + # Spec uses real github.com, which means outage of github.com can actually block deployment + # Keep spec in reliable bucket but don't run in blocking pipelines + RSpec.describe 'Manage', :github, :reliable, :skip_live_env, :requires_admin do describe 'Project import', issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/353583' do let!(:api_client) { Runtime::API::Client.as_admin } let!(:group) { Resource::Group.fabricate_via_api! { |resource| resource.api_client = api_client } } diff --git a/qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb b/qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb index 84eda023576..6585d08d2ac 100644 --- a/qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb +++ b/qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb @@ -21,11 +21,11 @@ module QA end end - let(:github_repo) { ENV['QA_LARGE_GH_IMPORT_REPO'] || 'rspec/rspec-core' } - let(:import_max_duration) { ENV['QA_LARGE_GH_IMPORT_DURATION'] ? ENV['QA_LARGE_GH_IMPORT_DURATION'].to_i : 7200 } + let(:github_repo) { ENV['QA_LARGE_IMPORT_REPO'] || 'rspec/rspec-core' } + let(:import_max_duration) { ENV['QA_LARGE_IMPORT_DURATION'] ? ENV['QA_LARGE_IMPORT_DURATION'].to_i : 7200 } let(:github_client) do Octokit::Client.new( - access_token: ENV['QA_LARGE_GH_IMPORT_GH_TOKEN'] || Runtime::Env.github_access_token, + access_token: ENV['QA_LARGE_IMPORT_GH_TOKEN'] || Runtime::Env.github_access_token, auto_paginate: true ) end @@ -106,37 +106,47 @@ module QA end end + # rubocop:disable RSpec/InstanceVariable after do |example| user.remove_via_api! unless example.exception next unless defined?(@import_time) - # save data for comparison after run finished + # save data for comparison notification creation save_json( "data", { + importer: :github, import_time: @import_time, reported_stats: @stats, - github: { + source: { + name: "GitHub", project_name: github_repo, - branches: gh_branches.length, - commits: gh_commits.length, - labels: gh_labels.length, - milestones: gh_milestones.length, - prs: gh_prs.length, - pr_comments: gh_prs.sum { |_k, v| v[:comments].length }, - issues: gh_issues.length, - issue_comments: gh_issues.sum { |_k, v| v[:comments].length } + address: "https://github.com", + data: { + branches: gh_branches.length, + commits: gh_commits.length, + labels: gh_labels.length, + milestones: gh_milestones.length, + mrs: gh_prs.length, + mr_comments: gh_prs.sum { |_k, v| v[:comments].length }, + issues: gh_issues.length, + issue_comments: gh_issues.sum { |_k, v| v[:comments].length } + } }, - gitlab: { + target: { + name: "GitLab", project_name: imported_project.path_with_namespace, - branches: gl_branches.length, - commits: gl_commits.length, - labels: gl_labels.length, - milestones: gl_milestones.length, - mrs: mrs.length, - mr_comments: mrs.sum { |_k, v| v[:comments].length }, - issues: gl_issues.length, - issue_comments: gl_issues.sum { |_k, v| v[:comments].length } + address: QA::Runtime::Scenario.gitlab_address, + data: { + branches: gl_branches.length, + commits: gl_commits.length, + labels: gl_labels.length, + milestones: gl_milestones.length, + mrs: mrs.length, + mr_comments: mrs.sum { |_k, v| v[:comments].length }, + issues: gl_issues.length, + issue_comments: gl_issues.sum { |_k, v| v[:comments].length } + } }, not_imported: { mrs: @mr_diff, @@ -145,6 +155,7 @@ module QA } ) end + # rubocop:enable RSpec/InstanceVariable it( 'imports large Github repo via api', @@ -153,7 +164,7 @@ module QA start = Time.now # import the project and log gitlab path - Runtime::Logger.info("== Importing project '#{github_repo}' in to '#{imported_project.reload!.full_path}' ==") + logger.info("== Importing project '#{github_repo}' in to '#{imported_project.reload!.full_path}' ==") # fetch all objects right after import has started fetch_github_objects diff --git a/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_large_project_spec.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_large_project_spec.rb new file mode 100644 index 00000000000..edb7838e81d --- /dev/null +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_large_project_spec.rb @@ -0,0 +1,412 @@ +# frozen_string_literal: true + +# rubocop:disable Rails/Pluck, Layout/LineLength, RSpec/MultipleMemoizedHelpers +module QA + RSpec.describe "Manage", requires_admin: 'uses admin API client for resource creation', + feature_flag: { name: 'bulk_import_projects', scope: :global }, + only: { job: 'large-gitlab-import' } do + describe "Gitlab migration" do + let(:logger) { Runtime::Logger.logger } + let(:differ) { RSpec::Support::Differ.new(color: true) } + let(:gitlab_group) { ENV['QA_LARGE_IMPORT_GROUP'] || 'gitlab-migration' } + let(:gitlab_project) { ENV['QA_LARGE_IMPORT_REPO'] || 'dri' } + let(:gitlab_source_address) { 'https://staging.gitlab.com' } + + let(:import_wait_duration) do + { + max_duration: (ENV['QA_LARGE_IMPORT_DURATION'] || 3600).to_i, + sleep_interval: 30 + } + end + + let(:admin_api_client) { Runtime::API::Client.as_admin } + + # explicitly create PAT via api to not create it via UI in environments where admin token env var is not present + let(:target_api_client) do + Runtime::API::Client.new( + user: user, + personal_access_token: Resource::PersonalAccessToken.fabricate_via_api! do |pat| + pat.api_client = admin_api_client + end.token + ) + end + + let(:user) do + Resource::User.fabricate_via_api! do |usr| + usr.api_client = admin_api_client + end + end + + let(:source_api_client) do + Runtime::API::Client.new( + gitlab_source_address, + personal_access_token: ENV["QA_LARGE_IMPORT_GL_TOKEN"], + is_new_session: false + ) + end + + let(:sandbox) do + Resource::Sandbox.fabricate_via_api! do |group| + group.api_client = admin_api_client + end + end + + let(:destination_group) do + Resource::Group.fabricate_via_api! do |group| + group.api_client = admin_api_client + group.sandbox = sandbox + group.path = "imported-group-destination-#{SecureRandom.hex(4)}" + end + end + + # Source group and it's objects + # + let(:source_group) do + Resource::Sandbox.fabricate_via_api! do |group| + group.api_client = source_api_client + group.path = gitlab_group + end + end + + let(:source_project) { source_group.projects.find { |project| project.name.include?(gitlab_project) }.reload! } + let(:source_branches) { source_project.repository_branches(auto_paginate: true).map { |b| b[:name] } } + let(:source_commits) { source_project.commits(auto_paginate: true).map { |c| c[:id] } } + let(:source_labels) { source_project.labels(auto_paginate: true).map { |l| l.except(:id) } } + let(:source_milestones) { source_project.milestones(auto_paginate: true).map { |ms| ms.except(:id, :web_url, :project_id) } } + let(:source_pipelines) { source_project.pipelines.map { |pp| pp.except(:id, :web_url, :project_id) } } + let(:source_mrs) { fetch_mrs(source_project, source_api_client) } + let(:source_issues) { fetch_issues(source_project, source_api_client) } + + # Imported group and it's objects + # + let(:imported_group) do + Resource::BulkImportGroup.fabricate_via_api! do |group| + group.import_access_token = source_api_client.personal_access_token # token for importing on source instance + group.api_client = target_api_client # token used by qa framework to access resources in destination instance + group.gitlab_address = gitlab_source_address + group.source_group = source_group + group.sandbox = destination_group + end + end + + let(:imported_project) { imported_group.projects.find { |project| project.name.include?(gitlab_project) }.reload! } + let(:branches) { imported_project.repository_branches(auto_paginate: true).map { |b| b[:name] } } + let(:commits) { imported_project.commits(auto_paginate: true).map { |c| c[:id] } } + let(:labels) { imported_project.labels(auto_paginate: true).map { |l| l.except(:id) } } + let(:milestones) { imported_project.milestones(auto_paginate: true).map { |ms| ms.except(:id, :web_url, :project_id) } } + let(:pipelines) { imported_project.pipelines.map { |pp| pp.except(:id, :web_url, :project_id) } } + let(:mrs) { fetch_mrs(imported_project, target_api_client) } + let(:issues) { fetch_issues(imported_project, target_api_client) } + + before do + Runtime::Feature.enable(:bulk_import_projects) + + destination_group.add_member(user, Resource::Members::AccessLevel::MAINTAINER) + end + + # rubocop:disable RSpec/InstanceVariable + after do |example| + next unless defined?(@import_time) + + # save data for comparison notification creation + save_json( + "data", + { + importer: :gitlab, + import_time: @import_time, + source: { + name: "GitLab Source", + project_name: source_project.path_with_namespace, + address: gitlab_source_address, + data: { + branches: source_branches.length, + commits: source_commits.length, + labels: source_labels.length, + milestones: source_milestones.length, + pipelines: source_pipelines.length, + mrs: source_mrs.length, + mr_comments: source_mrs.sum { |_k, v| v[:comments].length }, + issues: source_issues.length, + issue_comments: source_issues.sum { |_k, v| v[:comments].length } + } + }, + target: { + name: "GitLab Target", + project_name: imported_project.path_with_namespace, + address: QA::Runtime::Scenario.gitlab_address, + data: { + branches: branches.length, + commits: commits.length, + labels: labels.length, + milestones: milestones.length, + pipelines: pipelines.length, + mrs: mrs.length, + mr_comments: mrs.sum { |_k, v| v[:comments].length }, + issues: issues.length, + issue_comments: issues.sum { |_k, v| v[:comments].length } + } + }, + not_imported: { + mrs: @mr_diff, + issues: @issue_diff + } + } + ) + end + # rubocop:enable RSpec/InstanceVariable + + it "migrates large gitlab group via api", testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/358842' do + start = Time.now + + # trigger import and log imported group path + logger.info("== Importing group '#{gitlab_group}' in to '#{imported_group.full_path}' ==") + + # fetch all objects right after import has started + fetch_source_gitlab_objects + + # wait for import to finish and save import time + logger.info("== Waiting for import to be finished ==") + expect { imported_group.import_status }.to eventually_eq('finished').within(import_wait_duration) + @import_time = Time.now - start + + aggregate_failures do + verify_repository_import + verify_labels_import + verify_milestones_import + verify_pipelines_import + verify_merge_requests_import + verify_issues_import + end + end + + # Fetch source project objects for comparison + # + # @return [void] + def fetch_source_gitlab_objects + logger.info("== Fetching source group objects ==") + + source_branches + source_commits + source_labels + source_milestones + source_pipelines + source_mrs + source_issues + end + + # Verify repository imported correctly + # + # @return [void] + def verify_repository_import + logger.info("== Verifying repository import ==") + expect(imported_project.description).to eq(source_project.description) + expect(branches).to match_array(source_branches) + expect(commits).to match_array(source_commits) + end + + # Verify imported labels + # + # @return [void] + def verify_labels_import + logger.info("== Verifying label import ==") + expect(labels).to include(*source_labels) + end + + # Verify milestones import + # + # @return [void] + def verify_milestones_import + logger.info("== Verifying milestones import ==") + expect(milestones).to match_array(source_milestones) + end + + # Verify pipelines import + # + # @return [void] + def verify_pipelines_import + logger.info("== Verifying pipelines import ==") + expect(pipelines).to match_array(source_pipelines) + end + + # Verify imported merge requests and mr issues + # + # @return [void] + def verify_merge_requests_import + logger.info("== Verifying merge request import ==") + @mr_diff = verify_mrs_or_issues('mr') + end + + # Verify imported issues and issue comments + # + # @return [void] + def verify_issues_import + logger.info("== Verifying issue import ==") + @issue_diff = verify_mrs_or_issues('issue') + end + + # Verify imported mrs or issues and return missing items + # + # @param [String] type verification object, 'mr' or 'issue' + # @return [Hash] + def verify_mrs_or_issues(type) + # Compare length to have easy to read overview how many objects are missing + # + expected = type == 'mr' ? source_mrs : source_issues + actual = type == 'mr' ? mrs : issues + count_msg = "Expected to contain same amount of #{type}s. Source: #{expected.length}, Target: #{actual.length}" + expect(actual.length).to eq(expected.length), count_msg + + missing_comments = verify_comments(type, actual, expected) + + { + "#{type}s": (expected.keys - actual.keys).map { |it| actual[it].slice(:title, :url) }, + "#{type}_comments": missing_comments + } + end + + # Verify imported comments + # + # @param [String] type verification object, 'mrs' or 'issues' + # @param [Hash] actual + # @param [Hash] expected + # @return [Hash] + def verify_comments(type, actual, expected) + actual.each_with_object([]) do |(key, actual_item), missing_comments| + expected_item = expected[key] + title = actual_item[:title] + msg = "expected #{type} with title '#{title}' to have" + + # Print title in the error message to see which object is missing + # + expect(actual_item).to be_truthy, "#{msg} been imported" + next unless expected_item + + # Print difference in the description + # + expected_body = expected_item[:body] + actual_body = actual_item[:body] + body_msg = "#{msg} same description. diff:\n#{differ.diff(expected_body, actual_body)}" + expect(actual_body).to eq(expected_body), body_msg + + # Print difference in state + # + expected_state = expected_item[:state] + actual_state = actual_item[:state] + state_msg = "#{msg} same state. Source: #{expected_state}, Target: #{actual_state}" + expect(actual_state).to eq(expected_state), state_msg + + # Print amount difference first + # + expected_comments = expected_item[:comments] + actual_comments = actual_item[:comments] + comment_count_msg = <<~MSG + #{msg} same amount of comments. Source: #{expected_comments.length}, Target: #{actual_comments.length} + MSG + expect(actual_comments.length).to eq(expected_comments.length), comment_count_msg + expect(actual_comments).to match_array(expected_comments) + + # Save missing comments + # + comment_diff = expected_comments - actual_comments + next if comment_diff.empty? + + missing_comments << { + title: title, + target_url: actual_item[:url], + source_url: expected_item[:url], + missing_comments: comment_diff + } + end + end + + private + + # Project merge requests with comments + # + # @param [QA::Resource::Project] + # @param [Runtime::API::Client] client + # @return [Hash] + def fetch_mrs(project, client) + imported_mrs = project.merge_requests(auto_paginate: true, attempts: 2) + + Parallel.map(imported_mrs, in_threads: 4) do |mr| + resource = Resource::MergeRequest.init do |resource| + resource.project = project + resource.iid = mr[:iid] + resource.api_client = client + end + + [mr[:iid], { + url: mr[:web_url], + title: mr[:title], + body: sanitize_description(mr[:description]) || '', + state: mr[:state], + comments: resource + .comments(auto_paginate: true, attempts: 2) + .map { |c| sanitize_comment(c[:body]) } + }] + end.to_h + end + + # Project issues with comments + # + # @param [QA::Resource::Project] + # @param [Runtime::API::Client] client + # @return [Hash] + def fetch_issues(project, client) + imported_issues = project.issues(auto_paginate: true, attempts: 2) + + Parallel.map(imported_issues, in_threads: 4) do |issue| + resource = Resource::Issue.init do |issue_resource| + issue_resource.project = project + issue_resource.iid = issue[:iid] + issue_resource.api_client = client + end + + [issue[:iid], { + url: issue[:web_url], + title: issue[:title], + state: issue[:state], + body: sanitize_description(issue[:description]) || '', + comments: resource + .comments(auto_paginate: true, attempts: 2) + .map { |c| sanitize_comment(c[:body]) } + }] + end.to_h + end + + # Importer user mention pattern + # + # @return [Regex] + def created_by_pattern + @created_by_pattern ||= /\n\n \*By gitlab-migration on \S+ \(imported from GitLab\)\*/ + end + + # Remove added prefixes and legacy diff format from comments + # + # @param [String] body + # @return [String] + def sanitize_comment(body) + body&.gsub(created_by_pattern, "") + end + + # Remove created by prefix from descripion + # + # @param [String] body + # @return [String] + def sanitize_description(body) + body&.gsub(created_by_pattern, "") + end + + # Save json as file + # + # @param [String] name + # @param [Hash] json + # @return [void] + def save_json(name, json) + File.open("tmp/#{name}.json", "w") { |file| file.write(JSON.pretty_generate(json)) } + end + end + end +end +# rubocop:enable Rails/Pluck, Layout/LineLength, RSpec/MultipleMemoizedHelpers diff --git a/qa/qa/specs/features/api/1_manage/migration/gitlab_project_migration_common.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_project_migration_common.rb index 70f19e9f3d7..5a9cef6fded 100644 --- a/qa/qa/specs/features/api/1_manage/migration/gitlab_project_migration_common.rb +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_project_migration_common.rb @@ -3,7 +3,11 @@ module QA # Disable on live envs until bulk_import_projects toggle is on by default # Otherwise tests running in parallel can disable feature in the middle of other test - RSpec.shared_context 'with gitlab project migration', :requires_admin, :skip_live_env do + RSpec.shared_context 'with gitlab project migration', requires_admin: 'creates a user via API', + feature_flag: { + name: 'bulk_import_projects', + scope: :global + } do let(:source_project_with_readme) { false } let(:import_wait_duration) { { max_duration: 300, sleep_interval: 2 } } let(:admin_api_client) { Runtime::API::Client.as_admin } diff --git a/qa/qa/specs/features/api/3_create/gitaly/changing_repository_storage_spec.rb b/qa/qa/specs/features/api/3_create/gitaly/changing_repository_storage_spec.rb index 624ddbb68e1..cd1b7730fa9 100644 --- a/qa/qa/specs/features/api/3_create/gitaly/changing_repository_storage_spec.rb +++ b/qa/qa/specs/features/api/3_create/gitaly/changing_repository_storage_spec.rb @@ -2,7 +2,7 @@ module QA RSpec.describe 'Create' do - describe 'Changing Gitaly repository storage', :requires_admin do + describe 'Changing Gitaly repository storage', :requires_admin, except: { job: 'review-qa-*' } do praefect_manager = Service::PraefectManager.new shared_examples 'repository storage move' do @@ -11,12 +11,16 @@ module QA expect { project.change_repository_storage(destination_storage[:name]) }.not_to raise_error expect { praefect_manager.verify_storage_move(source_storage, destination_storage, repo_type: :project) }.not_to raise_error - Resource::Repository::ProjectPush.fabricate! do |push| - push.project = project - push.file_name = 'new_file' - push.file_content = '# This is a new file' - push.commit_message = 'Add new file' - push.new_branch = false + Support::Retrier.retry_on_exception(sleep_interval: 5) do + # For a short period of time after migrating, the repository can be 'read only' which may lead to errors + # 'The repository is temporarily read-only. Please try again later.' + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add new file' + commit.add_files([ + { file_path: 'new_file', content: '# This is a new file' } + ]) + end end expect(project).to have_file('README.md') @@ -45,7 +49,7 @@ module QA # Note: This test doesn't have the :orchestrated tag because it runs in the Test::Integration::Praefect # scenario with other tests that aren't considered orchestrated. # It also runs on staging using nfs-file07 as non-cluster storage and nfs-file22 as cluster/praefect storage - context 'when moving from Gitaly to Gitaly Cluster', :requires_praefect, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347828', quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/284645', type: :investigating } do + context 'when moving from Gitaly to Gitaly Cluster', :requires_praefect, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347828' do let(:source_storage) { { type: :gitaly, name: QA::Runtime::Env.non_cluster_repository_storage } } let(:destination_storage) { { type: :praefect, name: QA::Runtime::Env.praefect_repository_storage } } let(:project) do diff --git a/qa/qa/specs/features/api/3_create/integrations/webhook_events_spec.rb b/qa/qa/specs/features/api/3_create/integrations/webhook_events_spec.rb index 7a277d754c9..aae0329003b 100644 --- a/qa/qa/specs/features/api/3_create/integrations/webhook_events_spec.rb +++ b/qa/qa/specs/features/api/3_create/integrations/webhook_events_spec.rb @@ -70,7 +70,7 @@ module QA end end - it 'sends an issues and note event', 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349723' do + it 'sends an issues and note event', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349723' do setup_webhook(issues: true, note: true) do |webhook, smocker| issue = Resource::Issue.fabricate_via_api! do |issue_init| issue_init.project = webhook.project diff --git a/qa/qa/specs/features/api/3_create/repository/commit_to_templated_project_spec.rb b/qa/qa/specs/features/api/3_create/repository/commit_to_templated_project_spec.rb new file mode 100644 index 00000000000..c06912e0367 --- /dev/null +++ b/qa/qa/specs/features/api/3_create/repository/commit_to_templated_project_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Create' do + describe 'Create a new project from a template' do + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'templated-project' + project.template_name = 'dotnetcore' + end + end + + it 'commits via the api', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/357234' do + expect do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.update_files( + [ + { + file_path: '.gitlab-ci.yml', + content: 'script' + } + ] + ) + commit.add_files( + [ + { + file_path: 'foo', + content: 'bar' + } + ] + ) + end + end.not_to raise_exception + end + end + end +end diff --git a/qa/qa/specs/features/api/4_verify/remove_runner_spec.rb b/qa/qa/specs/features/api/4_verify/remove_runner_spec.rb index 0d10783735b..6e6198328e5 100644 --- a/qa/qa/specs/features/api/4_verify/remove_runner_spec.rb +++ b/qa/qa/specs/features/api/4_verify/remove_runner_spec.rb @@ -7,7 +7,7 @@ module QA let(:api_client) { Runtime::API::Client.new(:gitlab) } let(:executor) { "qa-runner-#{Time.now.to_i}" } - let(:runner_tags) { ['runner-registration-e2e-test'] } + let(:runner_tags) { ["runner-registration-e2e-test-#{Faker::Alphanumeric.alphanumeric(number: 8)}"] } let!(:runner) do Resource::Runner.fabricate! do |runner| runner.name = executor @@ -15,21 +15,19 @@ module QA end end - before do - sleep 5 # Runner should register within 5 seconds - end - # Removing a runner via the UI is covered by `spec/features/runners_spec.rb`` - it 'removes the runner', quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/355302', type: :investigating } do - expect(runner.project.runners.size).to eq(1) - expect(runner.project.runners.first[:description]).to eq(executor) + it 'removes the runner', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/354828' do + runners = nil + expect { (runners = runner.list_of_runners(tag_list: runner_tags)).size } + .to eventually_eq(1).within(max_duration: 10, sleep_interval: 1) + expect(runners.first[:description]).to eq(executor) - request = Runtime::API::Request.new(api_client, "runners/#{runner.project.runners.first[:id]}") + request = Runtime::API::Request.new(api_client, "runners/#{runners.first[:id]}") response = delete(request.url) expect(response.code).to eq(Support::API::HTTP_STATUS_NO_CONTENT) expect(response.body).to be_empty - expect(runner.project.runners).to be_empty + expect(runner.list_of_runners(tag_list: runner_tags)).to be_empty end end end diff --git a/qa/qa/specs/features/browser_ui/1_manage/login/log_into_mattermost_via_gitlab_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/login/log_into_mattermost_via_gitlab_spec.rb index 6bfb9c96fbd..80e660c1c1d 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/login/log_into_mattermost_via_gitlab_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/login/log_into_mattermost_via_gitlab_spec.rb @@ -11,7 +11,7 @@ module QA Page::Mattermost::Login.perform(&:sign_in_using_oauth) Page::Mattermost::Main.perform do |mattermost| - expect(mattermost).to have_content(/(Welcome to: Mattermost|Logout GitLab Mattermost)/) + expect(mattermost).to have_content(/(GitLab Mattermost|What’s the name of your organization)/) end end end diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/create_project_badge_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/create_project_badge_spec.rb index 2933d580957..3921595204c 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/project/create_project_badge_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/project/create_project_badge_spec.rb @@ -2,7 +2,7 @@ module QA RSpec.describe 'Manage' do - describe 'Create project badge' do + describe 'Create project badge', :reliable do let(:badge_name) { "project-badge-#{SecureRandom.hex(8)}" } let(:expected_badge_link_url) { "#{Runtime::Scenario.gitlab_address}/#{project.path_with_namespace}" } let(:expected_badge_image_url) { "#{Runtime::Scenario.gitlab_address}/#{project.path_with_namespace}/badges/main/pipeline.svg" } diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/import_github_repo_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/import_github_repo_spec.rb index d803f5e473c..3bf5a11b074 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/project/import_github_repo_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/project/import_github_repo_spec.rb @@ -1,7 +1,9 @@ # frozen_string_literal: true module QA - RSpec.describe 'Manage', :reliable, :github, :requires_admin do + # Spec uses real github.com, which means outage of github can actually block deployment + # Keep spec in reliable bucket but don't run in blocking pipelines + RSpec.describe 'Manage', :github, :reliable, :skip_live_env, :requires_admin do describe 'Project import' do let(:github_repo) { 'gitlab-qa-github/import-test' } let(:api_client) { Runtime::API::Client.as_admin } diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/personal_project_permissions_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/personal_project_permissions_spec.rb index 2aefa1c39ed..5d0befea1ce 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/project/personal_project_permissions_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/project/personal_project_permissions_spec.rb @@ -2,7 +2,7 @@ module QA RSpec.describe 'Manage' do - describe 'Personal project permissions' do + describe 'Personal project permissions', :reliable do let!(:owner) { Resource::User.fabricate_or_use(Runtime::Env.gitlab_qa_username_1, Runtime::Env.gitlab_qa_password_1) } let!(:owner_api_client) { Runtime::API::Client.new(:gitlab, user: owner) } diff --git a/qa/qa/specs/features/browser_ui/2_plan/issue/mentions_spec.rb b/qa/qa/specs/features/browser_ui/2_plan/issue/mentions_spec.rb index e7025920def..96f5731ea65 100644 --- a/qa/qa/specs/features/browser_ui/2_plan/issue/mentions_spec.rb +++ b/qa/qa/specs/features/browser_ui/2_plan/issue/mentions_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Plan', :smoke, :reliable do + RSpec.describe 'Plan', :smoke do describe 'mention' do let(:user) { Resource::User.fabricate_or_use(Runtime::Env.gitlab_qa_username_1, Runtime::Env.gitlab_qa_password_1) } let(:project) do diff --git a/qa/qa/specs/features/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb b/qa/qa/specs/features/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb index e5a6c57656e..ea531d84634 100644 --- a/qa/qa/specs/features/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb @@ -30,7 +30,7 @@ module QA setup_project_integration_with_jenkins - expect(page).to have_text("Jenkins CI activated.") + expect(page).to have_text("Jenkins settings saved and active.") QA::Support::Retrier.retry_on_exception do Resource::Repository::ProjectPush.fabricate! do |push| diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_via_template_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_via_template_spec.rb index c4aacd8fb06..3373f4f4233 100644 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_via_template_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_via_template_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Create' do + RSpec.describe 'Create', :reliable do describe 'Merge request custom templates' do let(:template_name) { 'custom_merge_request_template'} let(:template_content) { 'This is a custom merge request template test' } diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb index b0c6d01e8ca..d1e852979d0 100644 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb @@ -3,7 +3,7 @@ module QA RSpec.describe 'Create' do describe 'Merge request creation from fork', quarantine: { - only: { subdomain: %i[canary production] }, + only: :production, issue: "https://gitlab.com/gitlab-org/gitlab/-/issues/343801", type: :investigation } do diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/revert/revert_commit_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/revert/revert_commit_spec.rb index a2b27e294e6..8885163b5e3 100644 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/revert/revert_commit_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/revert/revert_commit_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Create' do + RSpec.describe 'Create', :reliable do describe 'Reverting a commit' do let(:file_name) { "secret_file.md" } diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/revert/reverting_merge_request_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/revert/reverting_merge_request_spec.rb index 90ca836f8b0..d66895de9c1 100644 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/revert/reverting_merge_request_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/revert/reverting_merge_request_spec.rb @@ -19,7 +19,7 @@ module QA Flow::Login.sign_in end - it 'can be reverted', :can_use_large_setup, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347709', quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/335987', type: :investigating } do + it 'can be reverted', :can_use_large_setup, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347709' do revertable_merge_request.visit! Page::MergeRequest::Show.perform do |merge_request| diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/file/create_file_via_web_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/file/create_file_via_web_spec.rb index f335cfdb367..095444d99f1 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/file/create_file_via_web_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/file/create_file_via_web_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Create' do + RSpec.describe 'Create', :reliable do context 'File management' do file_name = 'QA Test - File name' file_content = 'QA Test - File content' diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/file/edit_file_via_web_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/file/edit_file_via_web_spec.rb index 25c095d9eda..95e7a2a12d0 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/file/edit_file_via_web_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/file/edit_file_via_web_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Create' do + RSpec.describe 'Create', :reliable do context 'File management' do let(:file) { Resource::File.fabricate_via_api! } diff --git a/qa/qa/specs/features/browser_ui/3_create/snippet/create_personal_snippet_with_multiple_files_spec.rb b/qa/qa/specs/features/browser_ui/3_create/snippet/create_personal_snippet_with_multiple_files_spec.rb index ce99822b572..0560a5b125c 100644 --- a/qa/qa/specs/features/browser_ui/3_create/snippet/create_personal_snippet_with_multiple_files_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/snippet/create_personal_snippet_with_multiple_files_spec.rb @@ -2,7 +2,7 @@ module QA RSpec.describe 'Create' do - describe 'Multiple file snippet' do + describe 'Multiple file snippet', :reliable do let(:snippet) do Resource::Snippet.fabricate_via_browser_ui! do |snippet| snippet.title = 'Personal snippet with multiple files' diff --git a/qa/qa/specs/features/browser_ui/3_create/snippet/create_project_snippet_with_multiple_files_spec.rb b/qa/qa/specs/features/browser_ui/3_create/snippet/create_project_snippet_with_multiple_files_spec.rb index 70891ec72c7..77b3c4df7e1 100644 --- a/qa/qa/specs/features/browser_ui/3_create/snippet/create_project_snippet_with_multiple_files_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/snippet/create_project_snippet_with_multiple_files_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Create' do + RSpec.describe 'Create', :reliable do describe 'Multiple file snippet' do let(:snippet) do Resource::ProjectSnippet.fabricate_via_browser_ui! do |snippet| diff --git a/qa/qa/specs/features/browser_ui/3_create/snippet/delete_file_from_snippet_spec.rb b/qa/qa/specs/features/browser_ui/3_create/snippet/delete_file_from_snippet_spec.rb index b6092ef0c4c..e9339342386 100644 --- a/qa/qa/specs/features/browser_ui/3_create/snippet/delete_file_from_snippet_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/snippet/delete_file_from_snippet_spec.rb @@ -2,7 +2,7 @@ module QA RSpec.describe 'Create' do - describe 'Multiple file snippet' do + describe 'Multiple file snippet', :reliable do let(:personal_snippet) do Resource::Snippet.fabricate_via_api! do |snippet| snippet.title = 'Personal snippet to delete file from' diff --git a/qa/qa/specs/features/browser_ui/3_create/snippet/share_snippet_spec.rb b/qa/qa/specs/features/browser_ui/3_create/snippet/share_snippet_spec.rb index 6777c113f36..182a21a9377 100644 --- a/qa/qa/specs/features/browser_ui/3_create/snippet/share_snippet_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/snippet/share_snippet_spec.rb @@ -2,7 +2,7 @@ module QA RSpec.describe 'Create' do - describe 'Sharing snippets' do + describe 'Sharing snippets', :reliable do let(:snippet) do Resource::Snippet.fabricate! do |snippet| snippet.title = 'Shared snippet' diff --git a/qa/qa/specs/features/browser_ui/3_create/web_ide/open_fork_in_web_ide_spec.rb b/qa/qa/specs/features/browser_ui/3_create/web_ide/open_fork_in_web_ide_spec.rb index 653c0657c81..e9871a70560 100644 --- a/qa/qa/specs/features/browser_ui/3_create/web_ide/open_fork_in_web_ide_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/web_ide/open_fork_in_web_ide_spec.rb @@ -2,10 +2,13 @@ module QA RSpec.describe 'Create' do - describe 'Open a fork in Web IDE', quarantine: { + describe 'Open a fork in Web IDE', + # TODO: remove limitation to only run on main when the test is fixed + only: { pipeline: :main }, + quarantine: { issue: "https://gitlab.com/gitlab-org/gitlab/-/issues/351696", type: :flaky - } do + } do let(:parent_project) do Resource::Project.fabricate_via_api! do |project| project.name = 'parent-project' diff --git a/qa/qa/specs/features/browser_ui/4_verify/ci_variable/pipeline_with_protected_variable_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/ci_variable/pipeline_with_protected_variable_spec.rb index 7704111ea21..5bb60e64da5 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/ci_variable/pipeline_with_protected_variable_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/ci_variable/pipeline_with_protected_variable_spec.rb @@ -135,7 +135,7 @@ module QA def go_to_pipeline_job(user) Flow::Login.sign_in(as: user) project.visit! - Flow::Pipeline.visit_latest_pipeline(pipeline_condition: 'completed') + Flow::Pipeline.visit_latest_pipeline Page::Project::Pipeline::Show.perform do |pipeline| pipeline.click_job('job') diff --git a/qa/qa/specs/features/browser_ui/4_verify/ci_variable/ui_variable_inheritable_when_forward_pipeline_variables_true_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/ci_variable/ui_variable_inheritable_when_forward_pipeline_variables_true_spec.rb new file mode 100644 index 00000000000..496cc5f8a60 --- /dev/null +++ b/qa/qa/specs/features/browser_ui/4_verify/ci_variable/ui_variable_inheritable_when_forward_pipeline_variables_true_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module QA + # TODO: + # Remove FF :ci_trigger_forward_variables + # when https://gitlab.com/gitlab-org/gitlab/-/issues/355572 is closed + RSpec.describe 'Verify', :runner, feature_flag: { + name: 'ci_trigger_forward_variables', + scope: :global + } do + describe 'UI defined variable' do + include_context 'variable inheritance test prep' + + before do + add_ci_file(downstream1_project, [downstream1_ci_file]) + add_ci_file(upstream_project, [upstream_ci_file, upstream_child1_ci_file]) + + start_pipeline_with_variable + Page::Project::Pipeline::Show.perform do |show| + Support::Waiter.wait_until { show.passed? } + end + end + + it( + 'is inheritable when forward:pipeline_variables is true', + :aggregate_failures, + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/358197' + ) do + visit_job_page('child1', 'child1_job') + verify_job_log_shows_variable_value + + page.go_back + + visit_job_page('downstream1', 'downstream1_job') + verify_job_log_shows_variable_value + end + + def upstream_ci_file + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + stages: + - test + - deploy + + child1_trigger: + stage: test + trigger: + include: .child1-ci.yml + forward: + pipeline_variables: true + + downstream1_trigger: + stage: deploy + trigger: + project: #{downstream1_project.full_path} + forward: + pipeline_variables: true + YAML + } + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/4_verify/ci_variable/ui_variable_non_inheritable_when_forward_pipeline_variables_false_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/ci_variable/ui_variable_non_inheritable_when_forward_pipeline_variables_false_spec.rb new file mode 100644 index 00000000000..2a0aaf6d7a3 --- /dev/null +++ b/qa/qa/specs/features/browser_ui/4_verify/ci_variable/ui_variable_non_inheritable_when_forward_pipeline_variables_false_spec.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module QA + # TODO: + # Remove FF :ci_trigger_forward_variables + # when https://gitlab.com/gitlab-org/gitlab/-/issues/355572 is closed + RSpec.describe 'Verify', :runner, feature_flag: { + name: 'ci_trigger_forward_variables', + scope: :global + } do + describe 'UI defined variable' do + include_context 'variable inheritance test prep' + + before do + add_ci_file(downstream1_project, [downstream1_ci_file]) + add_ci_file(downstream2_project, [downstream2_ci_file]) + add_ci_file(upstream_project, [upstream_ci_file, upstream_child1_ci_file, upstream_child2_ci_file]) + + start_pipeline_with_variable + Page::Project::Pipeline::Show.perform do |show| + Support::Waiter.wait_until { show.passed? } + end + end + + it( + 'is not inheritable when forward:pipeline_variables is false', + :aggregate_failures, + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/358199' + ) do + visit_job_page('child1', 'child1_job') + verify_job_log_does_not_show_variable_value + + page.go_back + + visit_job_page('downstream1', 'downstream1_job') + verify_job_log_does_not_show_variable_value + end + + it( + 'is not inheritable by default', + :aggregate_failures, + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/358200' + ) do + visit_job_page('child2', 'child2_job') + verify_job_log_does_not_show_variable_value + + page.go_back + + visit_job_page('downstream2', 'downstream2_job') + verify_job_log_does_not_show_variable_value + end + + def upstream_ci_file + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + stages: + - test + - deploy + + child1_trigger: + stage: test + trigger: + include: .child1-ci.yml + forward: + pipeline_variables: false + + # default behavior + child2_trigger: + stage: test + trigger: + include: .child2-ci.yml + + downstream1_trigger: + stage: deploy + trigger: + project: #{downstream1_project.full_path} + forward: + pipeline_variables: false + + # default behavior + downstream2_trigger: + stage: deploy + trigger: + project: #{downstream2_project.full_path} + YAML + } + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb index bd3135bafdc..1bba5355790 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb @@ -1,7 +1,10 @@ # frozen_string_literal: true module QA - RSpec.describe 'Verify', :smoke, :runner do + RSpec.describe 'Verify', :smoke, :runner, quarantine: { + issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/356295', + type: :investigating + } do describe 'Pipeline creation and processing' do let(:executor) { "qa-runner-#{Time.now.to_i}" } @@ -58,6 +61,16 @@ module QA artifacts: paths: - my-artifacts/ + + test-coverage-report: + tags: + - #{executor} + script: mkdir coverage; echo "CONTENTS" > coverage/cobertura.xml + artifacts: + reports: + coverage_report: + coverage_format: cobertura + path: coverage/cobertura.xml YAML } ] @@ -71,7 +84,8 @@ module QA 'test-success': 'passed', 'test-failure': 'failed', 'test-tags-mismatch': 'pending', - 'test-artifacts': 'passed' + 'test-artifacts': 'passed', + 'test-coverage-report': 'passed' }.each do |job, status| Page::Project::Pipeline::Show.perform do |pipeline| pipeline.click_job(job) diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb index 9521cd20fc5..2fa6b9179ef 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb @@ -32,7 +32,7 @@ module QA add_included_files add_main_ci_file project.visit! - Flow::Pipeline.visit_latest_pipeline(pipeline_condition: 'succeeded') + Flow::Pipeline.visit_latest_pipeline(status: 'passed') end after do diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/locked_artifacts_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/locked_artifacts_spec.rb index 9abb25c8dc1..3356d1274c8 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/locked_artifacts_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/locked_artifacts_spec.rb @@ -56,7 +56,7 @@ module QA ) end.project.visit! - Flow::Pipeline.visit_latest_pipeline(pipeline_condition: 'completed') + Flow::Pipeline.visit_latest_pipeline Page::Project::Pipeline::Show.perform do |pipeline| pipeline.click_job('test-artifacts') diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/multi-project_pipelines_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/multi-project_pipelines_spec.rb new file mode 100644 index 00000000000..0d8756fc9a3 --- /dev/null +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/multi-project_pipelines_spec.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Verify' do + describe 'Multi-project pipelines' do + let(:downstream_job_name) { 'downstream_job' } + let(:executor) { "qa-runner-#{SecureRandom.hex(4)}" } + let!(:group) { Resource::Group.fabricate_via_api! } + + let(:upstream_project) do + Resource::Project.fabricate_via_api! do |project| + project.group = group + project.name = 'upstream-project' + end + end + + let(:downstream_project) do + Resource::Project.fabricate_via_api! do |project| + project.group = group + project.name = 'downstream-project' + end + end + + let!(:runner) do + Resource::Runner.fabricate_via_api! do |runner| + runner.token = group.reload!.runners_token + runner.name = executor + runner.tags = [executor] + end + end + + before do + add_ci_file(downstream_project, downstream_ci_file) + add_ci_file(upstream_project, upstream_ci_file) + + Flow::Login.sign_in + upstream_project.visit! + Flow::Pipeline.visit_latest_pipeline(status: 'passed') + end + + after do + runner.remove_via_api! + [upstream_project, downstream_project].each(&:remove_via_api!) + end + + it( + 'creates a multi-project pipeline', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/358064' + ) do + Page::Project::Pipeline::Show.perform do |show| + expect(show).to have_passed + expect(show).not_to have_job(downstream_job_name) + + show.expand_linked_pipeline + + expect(show).to have_job(downstream_job_name) + end + end + + private + + def add_ci_file(project, file) + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add CI config file' + commit.add_files([file]) + end + end + + def upstream_ci_file + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + stages: + - test + - deploy + + job1: + stage: test + tags: ["#{executor}"] + script: echo 'done' + + staging: + stage: deploy + trigger: + project: #{downstream_project.path_with_namespace} + strategy: depend + YAML + } + end + + def downstream_ci_file + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + "#{downstream_job_name}": + stage: test + tags: ["#{executor}"] + script: echo 'done' + YAML + } + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_dependent_relationship_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/parent_child_pipelines_dependent_relationship_spec.rb index e34f41b4c95..5b7a569fa8a 100644 --- a/qa/qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_dependent_relationship_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/parent_child_pipelines_dependent_relationship_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Release', :runner, :reliable do + RSpec.describe 'Verify', :runner, :reliable do describe 'Parent-child pipelines dependent relationship' do let!(:project) do Resource::Project.fabricate_via_api! do |project| @@ -25,23 +25,29 @@ module QA runner.remove_via_api! end - it 'parent pipelines passes if child passes', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348092' do + it( + 'parent pipelines passes if child passes', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/358062' + ) do add_ci_files(success_child_ci_file) - Flow::Pipeline.visit_latest_pipeline(pipeline_condition: 'completed') + Flow::Pipeline.visit_latest_pipeline Page::Project::Pipeline::Show.perform do |parent_pipeline| expect(parent_pipeline).to have_child_pipeline - expect(parent_pipeline).to have_passed + expect { parent_pipeline.has_passed? }.to eventually_be_truthy end end - it 'parent pipeline fails if child fails', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348091' do + it( + 'parent pipeline fails if child fails', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/358063' + ) do add_ci_files(fail_child_ci_file) - Flow::Pipeline.visit_latest_pipeline(pipeline_condition: 'completed') + Flow::Pipeline.visit_latest_pipeline Page::Project::Pipeline::Show.perform do |parent_pipeline| expect(parent_pipeline).to have_child_pipeline - expect(parent_pipeline).to have_failed + expect { parent_pipeline.has_failed? }.to eventually_be_truthy end end diff --git a/qa/qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_independent_relationship_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/parent_child_pipelines_independent_relationship_spec.rb index ef0c8d35c37..9e3c29db9e7 100644 --- a/qa/qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_independent_relationship_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/parent_child_pipelines_independent_relationship_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Release', :runner, :reliable do + RSpec.describe 'Verify', :runner, :reliable do describe 'Parent-child pipelines independent relationship' do let!(:project) do Resource::Project.fabricate_via_api! do |project| @@ -25,23 +25,29 @@ module QA runner.remove_via_api! end - it 'parent pipelines passes if child passes', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348093' do + it( + 'parent pipelines passes if child passes', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/358059' + ) do add_ci_files(success_child_ci_file) - Flow::Pipeline.visit_latest_pipeline(pipeline_condition: 'completed') + Flow::Pipeline.visit_latest_pipeline Page::Project::Pipeline::Show.perform do |parent_pipeline| expect(parent_pipeline).to have_child_pipeline - expect(parent_pipeline).to have_passed + expect { parent_pipeline.has_passed? }.to eventually_be_truthy end end - it 'parent pipeline passes even if child fails', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348094' do + it( + 'parent pipeline passes even if child fails', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/358060' + ) do add_ci_files(fail_child_ci_file) - Flow::Pipeline.visit_latest_pipeline(pipeline_condition: 'completed') + Flow::Pipeline.visit_latest_pipeline Page::Project::Pipeline::Show.perform do |parent_pipeline| expect(parent_pipeline).to have_child_pipeline - expect(parent_pipeline).to have_passed + expect { parent_pipeline.has_passed? }.to eventually_be_truthy end end diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/pass_dotenv_variables_to_downstream_via_bridge_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/pass_dotenv_variables_to_downstream_via_bridge_spec.rb index 0bc3fb7b829..bbcc71bade7 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/pass_dotenv_variables_to_downstream_via_bridge_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/pass_dotenv_variables_to_downstream_via_bridge_spec.rb @@ -3,8 +3,8 @@ module QA RSpec.describe 'Verify', :runner do describe 'Pass dotenv variables to downstream via bridge' do - let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" } - let(:upstream_var) { Faker::Alphanumeric.alphanumeric(8) } + let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" } + let(:upstream_var) { Faker::Alphanumeric.alphanumeric(number: 8) } let(:group) { Resource::Group.fabricate_via_api! } let(:upstream_project) do @@ -34,7 +34,7 @@ module QA add_ci_file(downstream_project, downstream_ci_file) add_ci_file(upstream_project, upstream_ci_file) upstream_project.visit! - Flow::Pipeline.visit_latest_pipeline(pipeline_condition: 'succeeded') + Flow::Pipeline.visit_latest_pipeline(status: 'passed') end after do @@ -44,8 +44,8 @@ module QA it 'runs the pipeline with composed config', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348088' do Page::Project::Pipeline::Show.perform do |parent_pipeline| - Support::Waiter.wait_until { parent_pipeline.has_child_pipeline? } - parent_pipeline.expand_child_pipeline + Support::Waiter.wait_until { parent_pipeline.has_linked_pipeline? } + parent_pipeline.expand_linked_pipeline parent_pipeline.click_job('downstream_test') end @@ -73,7 +73,7 @@ module QA stage: build tags: ["#{executor}"] script: - - echo "DYNAMIC_ENVIRONMENT_VAR=#{upstream_var}" >> variables.env + - for i in `seq 1 20`; do echo "VAR_$i=#{upstream_var}" >> variables.env; done; artifacts: reports: dotenv: variables.env @@ -81,7 +81,7 @@ module QA trigger: stage: deploy variables: - PASSED_MY_VAR: $DYNAMIC_ENVIRONMENT_VAR + PASSED_MY_VAR: "$VAR_#{rand(1..20)}" trigger: #{downstream_project.full_path} YAML } diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_can_create_merge_request_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_can_create_merge_request_spec.rb index 0e7a38626aa..b9f616aa733 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_can_create_merge_request_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_can_create_merge_request_spec.rb @@ -22,7 +22,7 @@ module QA it( 'can create merge request', - test_case: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349130' + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349130' ) do Page::Project::PipelineEditor::New.perform(&:create_new_ci) diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/run_pipeline_via_web_only_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/run_pipeline_via_web_only_spec.rb index d1e9981ae74..f36593218a9 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/run_pipeline_via_web_only_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/run_pipeline_via_web_only_spec.rb @@ -42,7 +42,7 @@ module QA it 'can trigger pipeline', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348011' do Page::Project::Pipeline::Index.perform do |index| - expect(index).not_to have_pipeline # should not auto trigger pipeline + expect(index).to have_no_pipeline # should not auto trigger pipeline index.click_run_pipeline_button end diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/run_pipeline_with_manual_jobs_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/run_pipeline_with_manual_jobs_spec.rb index 7a2c2b4ae90..fb7e3a8437f 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/run_pipeline_with_manual_jobs_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/run_pipeline_with_manual_jobs_spec.rb @@ -1,11 +1,10 @@ # frozen_string_literal: true module QA - RSpec.describe 'Verify', :runner, quarantine: { - type: :flaky, - issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/351994' - } do + RSpec.describe 'Verify', :runner do describe 'Run pipeline with manual jobs' do + let(:executor) { "qa-runner-#{SecureRandom.hex(4)}" } + let(:project) do Resource::Project.fabricate_via_api! do |project| project.name = 'pipeline-with-manual-job' @@ -16,7 +15,8 @@ module QA let!(:runner) do Resource::Runner.fabricate! do |runner| runner.project = project - runner.name = "qa-runner-#{SecureRandom.hex(3)}" + runner.name = executor + runner.tags = [executor] end end @@ -36,22 +36,26 @@ module QA Prep: stage: Stage1 + tags: ["#{executor}"] script: exit 0 when: manual Build: stage: Stage2 + tags: ["#{executor}"] needs: ['Prep'] script: exit 0 parallel: 6 Test: stage: Stage3 + tags: ["#{executor}"] needs: ['Build'] script: exit 0 Deploy: stage: Stage3 + tags: ["#{executor}"] needs: ['Test'] script: exit 0 parallel: 6 @@ -65,15 +69,17 @@ module QA before do Flow::Login.sign_in project.visit! - Flow::Pipeline.visit_latest_pipeline(pipeline_condition: 'skipped') + Flow::Pipeline.visit_latest_pipeline(status: 'skipped') end after do runner&.remove_via_api! - project&.remove_via_api! end - it 'does not leave any job in skipped state', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349158' do + it( + 'does not leave any job in skipped state', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349158' + ) do Page::Project::Pipeline::Show.perform do |show| show.click_job_action('Prep') # Trigger pipeline manually diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_child_pipeline_with_manual_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_child_pipeline_with_manual_spec.rb index ed46481d3be..1c75beebb48 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_child_pipeline_with_manual_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_child_pipeline_with_manual_spec.rb @@ -23,7 +23,7 @@ module QA Flow::Login.sign_in add_ci_files project.visit! - Flow::Pipeline.visit_latest_pipeline(pipeline_condition: 'succeeded') + Flow::Pipeline.visit_latest_pipeline(status: 'passed') end after do diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_matrix_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_matrix_spec.rb index 94ac857f0fe..205b4d1168a 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_matrix_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_matrix_spec.rb @@ -23,7 +23,7 @@ module QA Flow::Login.sign_in add_ci_files project.visit! - Flow::Pipeline.visit_latest_pipeline(pipeline_condition: 'succeeded') + Flow::Pipeline.visit_latest_pipeline(status: 'passed') end after do diff --git a/qa/qa/specs/features/browser_ui/4_verify/runner/register_runner_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/runner/register_runner_spec.rb index 8aa01888ae3..f8261bba342 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/runner/register_runner_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/runner/register_runner_spec.rb @@ -22,8 +22,6 @@ module QA Page::Project::Menu.perform(&:go_to_ci_cd_settings) Page::Project::Settings::CiCd.perform do |settings| - sleep 5 # Runner should register within 5 seconds - settings.expand_runners_settings do |page| expect(page).to have_content(executor) expect(page).to have_online_runner diff --git a/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_omnibus_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_omnibus_spec.rb index c833aa1a5b8..f570ad335fe 100644 --- a/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_omnibus_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_omnibus_spec.rb @@ -3,6 +3,8 @@ module QA RSpec.describe 'Package', :orchestrated, :skip_live_env do describe 'Self-managed Container Registry' do + include Support::Helpers::MaskToken + let(:project) do Resource::Project.fabricate_via_api! do |project| project.name = 'project-with-registry' @@ -110,9 +112,9 @@ module QA let(:auth_token) do case authentication_token_type when :personal_access_token - "\"#{personal_access_token}\"" + use_ci_variable(name: 'PERSONAL_ACCESS_TOKEN', value: personal_access_token, project: project) when :project_deploy_token - "\"#{project_deploy_token.token}\"" + use_ci_variable(name: 'PROJECT_DEPLOY_TOKEN', value: project_deploy_token.token, project: project) when :ci_job_token '$CI_JOB_TOKEN' end diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/composer_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/composer_registry_spec.rb index 2da0f6a0cf8..d5ef9dce10d 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/composer_registry_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/composer_registry_spec.rb @@ -8,6 +8,7 @@ module QA let(:project) do Resource::Project.fabricate_via_api! do |project| project.name = 'composer-package-project' + project.visibility = :private end end diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/conan_repository_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/conan_repository_spec.rb index 22495796605..1840ae4e7f8 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/conan_repository_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/conan_repository_spec.rb @@ -12,6 +12,7 @@ module QA let(:project) do Resource::Project.fabricate_via_api! do |project| project.name = 'conan-package-project' + project.visibility = :private end end diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/generic_repository_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/generic_repository_spec.rb index 71acc3a8f92..e37102c17f7 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/generic_repository_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/generic_repository_spec.rb @@ -8,6 +8,7 @@ module QA let(:project) do Resource::Project.fabricate_via_api! do |project| project.name = 'generic-package-project' + project.visibility = :private end end diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/helm_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/helm_registry_spec.rb index d2e816f9bf9..078465770c6 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/helm_registry_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/helm_registry_spec.rb @@ -5,6 +5,7 @@ module QA describe 'Helm Registry' do using RSpec::Parameterized::TableSyntax include Runtime::Fixtures + include Support::Helpers::MaskToken include_context 'packages registry qa scenario' let(:package_name) { "gitlab_qa_helm-#{SecureRandom.hex(8)}" } @@ -32,11 +33,13 @@ module QA let(:access_token) do case authentication_token_type when :personal_access_token - personal_access_token + use_ci_variable(name: 'PERSONAL_ACCESS_TOKEN', value: personal_access_token, project: package_project) + use_ci_variable(name: 'PERSONAL_ACCESS_TOKEN', value: personal_access_token, project: client_project) when :ci_job_token '${CI_JOB_TOKEN}' when :project_deploy_token - project_deploy_token.token + use_ci_variable(name: 'PROJECT_DEPLOY_TOKEN', value: project_deploy_token.token, project: package_project) + use_ci_variable(name: 'PROJECT_DEPLOY_TOKEN', value: project_deploy_token.token, project: client_project) end end diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb index 04aaefbaf5c..0ee5f1b6a0b 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb @@ -5,6 +5,7 @@ module QA describe 'npm instance level endpoint' do using RSpec::Parameterized::TableSyntax include Runtime::Fixtures + include Support::Helpers::MaskToken let!(:registry_scope) { Runtime::Namespace.sandbox_name } let!(:personal_access_token) do @@ -78,11 +79,13 @@ module QA let(:auth_token) do case authentication_token_type when :personal_access_token - "\"#{personal_access_token}\"" + use_ci_variable(name: 'PERSONAL_ACCESS_TOKEN', value: personal_access_token, project: project) + use_ci_variable(name: 'PERSONAL_ACCESS_TOKEN', value: personal_access_token, project: another_project) when :ci_job_token '${CI_JOB_TOKEN}' when :project_deploy_token - "\"#{project_deploy_token.token}\"" + use_ci_variable(name: 'PROJECT_DEPLOY_TOKEN', value: project_deploy_token.token, project: project) + use_ci_variable(name: 'PROJECT_DEPLOY_TOKEN', value: project_deploy_token.token, project: another_project) end end diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb index cad1802f3e9..5ebcb94d0d0 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb @@ -5,6 +5,7 @@ module QA describe 'npm project level endpoint' do using RSpec::Parameterized::TableSyntax include Runtime::Fixtures + include Support::Helpers::MaskToken let!(:registry_scope) { Runtime::Namespace.sandbox_name } let!(:personal_access_token) do @@ -34,6 +35,7 @@ module QA let!(:project) do Resource::Project.fabricate_via_api! do |project| project.name = 'npm-project-level' + project.visibility = :private end end @@ -69,11 +71,11 @@ module QA let(:auth_token) do case authentication_token_type when :personal_access_token - "\"#{personal_access_token}\"" + use_ci_variable(name: 'PERSONAL_ACCESS_TOKEN', value: personal_access_token, project: project) when :ci_job_token '${CI_JOB_TOKEN}' when :project_deploy_token - "\"#{project_deploy_token.token}\"" + use_ci_variable(name: 'PROJECT_DEPLOY_TOKEN', value: project_deploy_token.token, project: project) end end diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_group_level_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_group_level_spec.rb index b0a6555a16b..0ddb59d6625 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_group_level_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_group_level_spec.rb @@ -5,6 +5,7 @@ module QA describe 'NuGet group level endpoint' do using RSpec::Parameterized::TableSyntax include Runtime::Fixtures + include Support::Helpers::MaskToken let(:project) do Resource::Project.fabricate_via_api! do |project| @@ -61,6 +62,8 @@ module QA after do runner.remove_via_api! package.remove_via_api! + project.remove_via_api! + another_project.remove_via_api! end where(:case_name, :authentication_token_type, :token_name, :testcase) do @@ -73,11 +76,13 @@ module QA let(:auth_token_password) do case authentication_token_type when :personal_access_token - "\"#{personal_access_token.token}\"" + use_ci_variable(name: 'PERSONAL_ACCESS_TOKEN', value: personal_access_token.token, project: project) + use_ci_variable(name: 'PERSONAL_ACCESS_TOKEN', value: personal_access_token.token, project: another_project) when :ci_job_token '${CI_JOB_TOKEN}' when :group_deploy_token - "\"#{group_deploy_token.token}\"" + use_ci_variable(name: 'GROUP_DEPLOY_TOKEN', value: group_deploy_token.token, project: project) + use_ci_variable(name: 'GROUP_DEPLOY_TOKEN', value: group_deploy_token.token, project: another_project) end end diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_project_level_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_project_level_spec.rb index 4cac055634e..d5fd78480d2 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_project_level_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_project_level_spec.rb @@ -3,6 +3,8 @@ module QA RSpec.describe 'Package', :orchestrated, :packages, :object_storage do describe 'NuGet project level endpoint' do + include Support::Helpers::MaskToken + let(:project) do Resource::Project.fabricate_via_api! do |project| project.name = 'nuget-package-project' @@ -77,11 +79,11 @@ module QA let(:auth_token_password) do case authentication_token_type when :personal_access_token - "\"#{personal_access_token.token}\"" + use_ci_variable(name: 'PERSONAL_ACCESS_TOKEN', value: personal_access_token.token, project: project) when :ci_job_token '${CI_JOB_TOKEN}' when :project_deploy_token - "\"#{project_deploy_token.token}\"" + use_ci_variable(name: 'PROJECT_DEPLOY_TOKEN', value: project_deploy_token.token, project: project) end end diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/pypi_repository_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/pypi_repository_spec.rb index a0c2eca5bd2..4614eced300 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/pypi_repository_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/pypi_repository_spec.rb @@ -4,10 +4,12 @@ module QA RSpec.describe 'Package', :orchestrated, :packages, :object_storage do describe 'PyPI Repository' do include Runtime::Fixtures + include Support::Helpers::MaskToken let(:project) do Resource::Project.fabricate_via_api! do |project| project.name = 'pypi-package-project' + project.visibility = :private end end @@ -30,7 +32,7 @@ module QA let(:uri) { URI.parse(Runtime::Scenario.gitlab_address) } let(:gitlab_address_with_port) { "#{uri.scheme}://#{uri.host}:#{uri.port}" } let(:gitlab_host_with_port) { "#{uri.host}:#{uri.port}" } - let(:personal_access_token) { Runtime::Env.personal_access_token } + let(:personal_access_token) { use_ci_variable(name: 'PERSONAL_ACCESS_TOKEN', value: Runtime::Env.personal_access_token, project: project) } before do Flow::Login.sign_in diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/rubygems_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/rubygems_registry_spec.rb index b2208dc644c..409a1c10943 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/rubygems_registry_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/rubygems_registry_spec.rb @@ -1,13 +1,15 @@ # frozen_string_literal: true module QA - RSpec.describe 'Package', :orchestrated, :requires_admin, :packages, :object_storage do + RSpec.describe 'Package', :orchestrated, :packages, :object_storage, + feature_flag: { name: 'rubygem_packages', scope: :project } do describe 'RubyGems Repository' do include Runtime::Fixtures let(:project) do Resource::Project.fabricate_via_api! do |project| project.name = 'rubygems-package-project' + project.visibility = :private end end diff --git a/qa/qa/specs/helpers/feature_flag.rb b/qa/qa/specs/helpers/feature_flag.rb new file mode 100644 index 00000000000..b9de2332c19 --- /dev/null +++ b/qa/qa/specs/helpers/feature_flag.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'rspec/core' + +module QA + module Specs + module Helpers + module FeatureFlag + extend self + + def skip_or_run_feature_flag_tests_or_contexts(example) + if example.metadata.key?(:feature_flag) + feature_flag_tag = example.metadata[:feature_flag] + + global_feature_flag_message = 'Skipping on .com environments due to global feature flag usage' + feature_flag_message = 'Skipping on production due to feature flag usage' + + if feature_flag_tag.is_a?(Hash) && feature_flag_tag[:scope] == :global + # Tests using a global feature flag will be skipped on live .com environments. + # This is to avoid flakiness with other tests running in parallel on the same environment + # as well as interfering with feature flag experimentation done by development groups. + example.metadata[:skip] = global_feature_flag_message if ContextSelector.dot_com? + else + # Tests using a feature flag scoped to an actor (ex: :project, :user, :group), or + # with no scope defined (such as in the case of a low risk global feature flag), + # will only be skipped in canary and production due to no admin account existing there. + example.metadata[:skip] = feature_flag_message if ContextSelector.context_matches?(:production) + end + end + end + end + end + end +end diff --git a/qa/qa/specs/runner.rb b/qa/qa/specs/runner.rb index 2b9adf0e870..c30e5d822c4 100644 --- a/qa/qa/specs/runner.rb +++ b/qa/qa/specs/runner.rb @@ -52,6 +52,8 @@ module QA tags_for_rspec end + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity def perform args = [] args.push('--tty') if tty @@ -89,12 +91,29 @@ module QA File.open(filename, 'w') { |f| f.write(total_examples) } if total_examples.to_i > 0 $stdout.puts "Total examples in #{Runtime::Scenario.klass}: #{total_examples}#{total_examples.to_i > 0 ? ". Saved to file: #{filename}" : ''}" + elsif Runtime::Scenario.attributes[:test_metadata_only] + args.unshift('--dry-run') + + output_file = Pathname.new(File.join(Runtime::Path.qa_root, 'tmp', 'test-metadata.json')) + + RSpec.configure do |config| + config.add_formatter(QA::Support::JsonFormatter, output_file) + config.fail_if_no_examples = true + end + + RSpec::Core::Runner.run(args.flatten, $stderr, $stdout) do |status| + abort if status.nonzero? + end + + $stdout.puts "Saved to file: #{output_file}" else RSpec::Core::Runner.run(args.flatten, *DEFAULT_STD_ARGS).tap do |status| abort if status.nonzero? end end end + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/CyclomaticComplexity private diff --git a/qa/qa/support/api.rb b/qa/qa/support/api.rb index 976188e45c6..0c0a1a90ff2 100644 --- a/qa/qa/support/api.rb +++ b/qa/qa/support/api.rb @@ -20,9 +20,7 @@ module QA verify_ssl: false } - RestClient::Request.execute( - default_args.merge(args) - ) + RestClient::Request.execute(default_args.merge(args)) rescue RestClient::ExceptionWithResponse => e return_response_or_raise(e) end @@ -56,13 +54,16 @@ module QA end end - def put(url, payload = nil) + def put(url, payload = nil, args = {}) with_retry_on_too_many_requests do - RestClient::Request.execute( + default_args = { method: :put, url: url, payload: payload, - verify_ssl: false) + verify_ssl: false + } + + RestClient::Request.execute(default_args.merge(args)) rescue RestClient::ExceptionWithResponse => e return_response_or_raise(e) end diff --git a/qa/qa/support/formatters/feature_flag_formatter.rb b/qa/qa/support/formatters/feature_flag_formatter.rb new file mode 100644 index 00000000000..94c039586f9 --- /dev/null +++ b/qa/qa/support/formatters/feature_flag_formatter.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module QA + module Support + module Formatters + class FeatureFlagFormatter < ::RSpec::Core::Formatters::BaseFormatter + include Specs::Helpers::FeatureFlag + + ::RSpec::Core::Formatters.register( + self, + :example_group_started, + :example_started + ) + + # Starts example group + # @param [RSpec::Core::Notifications::GroupNotification] example_group_notification + # @return [void] + def example_group_started(example_group_notification) + group = example_group_notification.group + + skip_or_run_feature_flag_tests_or_contexts(group) + end + + # Starts example + # @param [RSpec::Core::Notifications::ExampleNotification] example_notification + # @return [void] + def example_started(example_notification) + example = example_notification.example + + # if skip propagated from example_group, do not reset skip metadata + skip_or_run_feature_flag_tests_or_contexts(example) unless example.metadata[:skip] + end + end + end + end +end diff --git a/qa/qa/support/formatters/test_stats_formatter.rb b/qa/qa/support/formatters/test_stats_formatter.rb index 16fc0a50b1b..9d19c2e8bb5 100644 --- a/qa/qa/support/formatters/test_stats_formatter.rb +++ b/qa/qa/support/formatters/test_stats_formatter.rb @@ -64,6 +64,7 @@ module QA name: example.full_description, file_path: file_path, status: example.execution_result.status, + smoke: example.metadata.key?(:smoke).to_s, reliable: example.metadata.key?(:reliable).to_s, quarantined: quarantined(example.metadata), retried: ((example.metadata[:retry_attempts] || 0) > 0).to_s, diff --git a/qa/qa/support/helpers/mask_token.rb b/qa/qa/support/helpers/mask_token.rb new file mode 100644 index 00000000000..1f8161f7173 --- /dev/null +++ b/qa/qa/support/helpers/mask_token.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module QA + module Support + module Helpers + module MaskToken + def use_ci_variable(name:, value:, project:) + Resource::CiVariable.fabricate_via_api! do |ci_variable| + ci_variable.project = project + ci_variable.key = name + ci_variable.value = value + ci_variable.protected = true + end + "$#{name}" + end + end + end + end +end diff --git a/qa/qa/support/helpers/plan.rb b/qa/qa/support/helpers/plan.rb index 298a6d3f036..b6950c6bacd 100644 --- a/qa/qa/support/helpers/plan.rb +++ b/qa/qa/support/helpers/plan.rb @@ -57,8 +57,9 @@ module QA }.freeze LICENSE_TYPE = { - license_file: 'license file', - cloud_license: 'cloud license' + legacy_license: 'legacy license', + online_cloud: 'online license', + offline_cloud: 'offline license' }.freeze end end diff --git a/qa/qa/support/loglinking.rb b/qa/qa/support/loglinking.rb index 89519e9537c..caf381912d3 100644 --- a/qa/qa/support/loglinking.rb +++ b/qa/qa/support/loglinking.rb @@ -5,7 +5,7 @@ module QA # Static address variables declared for mapping environment to logging URLs STAGING_ADDRESS = 'https://staging.gitlab.com' STAGING_REF_ADDRESS = 'https://staging-ref.gitlab.com' - PRODUCTION_ADDRESS = 'https://www.gitlab.com' + PRODUCTION_ADDRESS = 'https://gitlab.com' PRE_PROD_ADDRESS = 'https://pre.gitlab.com' SENTRY_ENVIRONMENTS = { staging: 'https://sentry.gitlab.net/gitlab/staginggitlabcom/?environment=gstg', @@ -30,7 +30,7 @@ module QA errors = ["Correlation Id: #{correlation_id}"] errors << "Sentry Url: #{sentry_uri}&query=correlation_id%3A%22#{correlation_id}%22" if sentry_uri - errors << "Kibana Url: #{kibana_uri}app/discover#/?_a=(query:(language:kuery,query:'json.correlation_id%20:%20#{correlation_id}'))" if kibana_uri + errors << "Kibana Url: #{kibana_uri}app/discover#/?_a=(query:(language:kuery,query:'json.correlation_id%20:%20#{correlation_id}'))&_g=(time:(from:now-24h%2Fh,to:now))" if kibana_uri errors.join("\n") end diff --git a/qa/qa/support/matchers/have_matcher.rb b/qa/qa/support/matchers/have_matcher.rb index a90d2df96ae..b96566a9e5d 100644 --- a/qa/qa/support/matchers/have_matcher.rb +++ b/qa/qa/support/matchers/have_matcher.rb @@ -10,6 +10,7 @@ module QA file_content assignee child_pipeline + linked_pipeline content design file diff --git a/qa/qa/support/page_error_checker.rb b/qa/qa/support/page_error_checker.rb index ede9b49bda6..192b8c147cd 100644 --- a/qa/qa/support/page_error_checker.rb +++ b/qa/qa/support/page_error_checker.rb @@ -49,7 +49,7 @@ module QA error_code = 404 if Nokogiri::HTML.parse(page.html).xpath("//img").map { |t| t[:alt] }.first.eql?('404') # 500 error page in header surrounded by newlines, try to match - five_hundred_test = Nokogiri::HTML.parse(page.html).xpath("//h1").map.first + five_hundred_test = Nokogiri::HTML.parse(page.html).xpath("//h1/text()").map.first unless five_hundred_test.nil? error_code = 500 if five_hundred_test.text.include?('500') end @@ -61,6 +61,39 @@ module QA end end + # Log request errors triggered from async api calls from the browser + # + # If any errors are found in the session, log them + # using QA::Runtime::Logger + # @param [Capybara::Session] page + def log_request_errors(page) + return if QA::Runtime::Browser.blank_page? + + url = page.driver.browser.current_url + QA::Runtime::Logger.debug "Fetching API error cache for #{url}" + + cache = page.execute_script <<~JS + return !(typeof(Interceptor)==="undefined") ? Interceptor.getCache() : null; + JS + + return unless cache&.dig('errors') + + grouped_errors = group_errors(cache['errors']) + + errors = grouped_errors.map do |error_metadata, request_id_string| + "#{error_metadata} -- #{request_id_string}" + end + + unless errors.nil? || errors.empty? + QA::Runtime::Logger.error "Interceptor Api Errors\n#{errors.join("\n")}" + end + + # clear the cache after logging the errors + page.execute_script <<~JS + Interceptor && Interceptor.saveCache({}); + JS + end + def error_report_for(logs) logs .map(&:message) @@ -70,6 +103,16 @@ module QA def logs(page) page.driver.browser.manage.logs.get(:browser) end + + private + + def group_errors(errors) + errors.each_with_object({}) do |error, memo| + url = error['url']&.split('?')&.first || 'Unknown url' + key = "[#{error['status']}] #{error['method']} #{url}" + memo[key] = "Correlation Id: #{error.dig('headers', 'x-request-id') || 'Correlation Id not found'}" + end + end end end end diff --git a/qa/qa/support/wait_for_requests.rb b/qa/qa/support/wait_for_requests.rb index 16af4bae521..89674a1d5c6 100644 --- a/qa/qa/support/wait_for_requests.rb +++ b/qa/qa/support/wait_for_requests.rb @@ -16,12 +16,16 @@ module QA Waiter.wait_until(log: false) do finished_all_ajax_requests? && (!skip_finished_loading_check ? finished_loading?(wait: 1) : true) end + QA::Support::PageErrorChecker.log_request_errors(Capybara.page) if QA::Runtime::Env.can_intercept? rescue Repeater::WaitExceededError raise $!, 'Page did not fully load. This could be due to an unending async request or loading icon.' end def finished_all_ajax_requests? - Capybara.page.evaluate_script('window.pendingRequests || window.pendingRailsUJSRequests || 0').zero? # rubocop:disable Style/NumericPredicate + requests = %w[window.pendingRequests window.pendingRailsUJSRequests 0] + requests.unshift('(window.Interceptor && window.Interceptor.activeFetchRequests)') if Runtime::Env.can_intercept? + script = requests.join(' || ') + Capybara.page.evaluate_script(script).zero? # rubocop:disable Style/NumericPredicate end def finished_loading?(wait: DEFAULT_MAX_WAIT_TIME) diff --git a/qa/qa/tools/reliable_report.rb b/qa/qa/tools/reliable_report.rb index 96e5690ce30..b3df6de3d54 100644 --- a/qa/qa/tools/reliable_report.rb +++ b/qa/qa/tools/reliable_report.rb @@ -11,6 +11,8 @@ module QA include Support::InfluxdbTools include Support::API + RELIABLE_REPORT_LABEL = "reliable test report" + # Project for report creation: https://gitlab.com/gitlab-org/gitlab PROJECT_ID = 278964 @@ -28,7 +30,11 @@ module QA reporter = new(range) reporter.print_report - reporter.report_in_issue_and_slack if report_in_issue_and_slack == "true" + + if report_in_issue_and_slack == "true" + reporter.report_in_issue_and_slack + reporter.close_previous_reports + end rescue StandardError => e reporter&.notify_failure(e) raise(e) @@ -51,16 +57,15 @@ module QA # @return [void] def report_in_issue_and_slack puts "Creating report".colorize(:green) - response = post( - "#{gitlab_api_url}/projects/#{PROJECT_ID}/issues", - { - title: "Reliable e2e test report", - description: report_issue_body, - labels: "Quality,test,type::maintenance,reliable test report,automation:ml" - }, - headers: { "PRIVATE-TOKEN" => gitlab_access_token } + issue = api_update( + :post, + "projects/#{PROJECT_ID}/issues", + title: "Reliable e2e test report", + description: report_issue_body, + labels: "#{RELIABLE_REPORT_LABEL},Quality,test,type::maintenance,automation:ml" ) - web_url = parse_body(response)[:web_url] + @report_iid = issue[:iid] + web_url = issue[:web_url] puts "Created report issue: #{web_url}" puts "Sending slack notification".colorize(:green) @@ -76,6 +81,25 @@ module QA puts "Done!" end + # Close previous reliable test reports + # + # @return [void] + def close_previous_reports + puts "Closing previous reports".colorize(:green) + issues = api_get("projects/#{PROJECT_ID}/issues?labels=#{RELIABLE_REPORT_LABEL}&state=opened") + + issues + .reject { |issue| issue[:iid] == report_iid } + .each do |issue| + issue_iid = issue[:iid] + issue_endpoint = "projects/#{PROJECT_ID}/issues/#{issue_iid}" + + puts "Closing previous report '#{issue[:web_url]}'" + api_update(:put, issue_endpoint, state_event: "close") + api_update(:post, "#{issue_endpoint}/notes", body: "Closed issue in favor of ##{report_iid}") + end + end + # Notify failure # # @param [StandardError] error @@ -89,7 +113,39 @@ module QA private - attr_reader :range, :slack_channel + attr_reader :range, :slack_channel, :report_iid + + # Slack notifier + # + # @return [Slack::Notifier] + def notifier + @notifier ||= Slack::Notifier.new( + slack_webhook_url, + channel: slack_channel, + username: "Reliable Spec Report" + ) + end + + # Gitlab access token + # + # @return [String] + def gitlab_access_token + @gitlab_access_token ||= ENV["GITLAB_ACCESS_TOKEN"] || raise("Missing GITLAB_ACCESS_TOKEN env variable") + end + + # Gitlab api url + # + # @return [String] + def gitlab_api_url + @gitlab_api_url ||= ENV["CI_API_V4_URL"] || raise("Missing CI_API_V4_URL env variable") + end + + # Slack webhook url + # + # @return [String] + def slack_webhook_url + @slack_webhook_url ||= ENV["SLACK_WEBHOOK"] || raise("Missing SLACK_WEBHOOK env variable") + end # Markdown formatted report issue body # @@ -316,6 +372,7 @@ module QA |> filter(fn: (r) => r.status != "pending" and r.merge_request == "false" and r.quarantined == "false" and + r.smoke == "false" and r.reliable == "#{reliable}" and r._field == "id" ) @@ -323,36 +380,30 @@ module QA QUERY end - # Slack notifier + # Api get request # - # @return [Slack::Notifier] - def notifier - @notifier ||= Slack::Notifier.new( - slack_webhook_url, - channel: slack_channel, - username: "Reliable Spec Report" - ) + # @param [String] path + # @param [Hash] payload + # @return [Hash, Array] + def api_get(path) + response = get("#{gitlab_api_url}/#{path}", { headers: { "PRIVATE-TOKEN" => gitlab_access_token } }) + parse_body(response) end - # Gitlab access token + # Api update request # - # @return [String] - def gitlab_access_token - @gitlab_access_token ||= ENV["GITLAB_ACCESS_TOKEN"] || raise("Missing GITLAB_ACCESS_TOKEN env variable") - end - - # Gitlab api url - # - # @return [String] - def gitlab_api_url - @gitlab_api_url ||= ENV["CI_API_V4_URL"] || raise("Missing CI_API_V4_URL env variable") - end - - # Slack webhook url - # - # @return [String] - def slack_webhook_url - @slack_webhook_url ||= ENV["SLACK_WEBHOOK"] || raise("Missing SLACK_WEBHOOK env variable") + # @param [Symbol] verb :post or :put + # @param [String] path + # @param [Hash] payload + # @return [Hash, Array] + def api_update(verb, path, **payload) + response = send( + verb, + "#{gitlab_api_url}/#{path}", + payload, + { headers: { "PRIVATE-TOKEN" => gitlab_access_token } } + ) + parse_body(response) end end end diff --git a/qa/qa/tools/test_resources_handler.rb b/qa/qa/tools/test_resources_handler.rb index 476f87fff6b..5218e6df217 100644 --- a/qa/qa/tools/test_resources_handler.rb +++ b/qa/qa/tools/test_resources_handler.rb @@ -75,11 +75,15 @@ module QA # Download files from GCS bucket by environment name # Delete the files afterward def download(ci_project_name) - files_list = gcs_storage.list_objects(BUCKET, prefix: ci_project_name).items.each_with_object([]) do |obj, arr| + bucket_items = gcs_storage.list_objects(BUCKET, prefix: ci_project_name).items + + files_list = bucket_items&.each_with_object([]) do |obj, arr| arr << obj.name end - return puts "\nNothing to download!" if files_list.empty? + return puts "\nNothing to download!" if files_list.blank? + + FileUtils.mkdir_p('tmp/') files_list.each do |file_name| local_path = "tmp/#{file_name.split('/').last}" diff --git a/qa/qa/vendor/jenkins/page/configure_job.rb b/qa/qa/vendor/jenkins/page/configure_job.rb index 471567ec828..65719795720 100644 --- a/qa/qa/vendor/jenkins/page/configure_job.rb +++ b/qa/qa/vendor/jenkins/page/configure_job.rb @@ -15,6 +15,7 @@ module QA def configure(scm_url:) set_git_source_code_management_url(scm_url) + set_git_branches_to_build("*/#{Runtime::Env.default_branch}") click_build_when_change_is_pushed_to_gitlab set_publish_status_to_gitlab @@ -31,6 +32,10 @@ module QA set_repository_url(repository_url) end + def set_git_branches_to_build(branches) + find('.setting-name', text: "Branch Specifier (blank for 'any')").find(:xpath, "..").find('input').set branches + end + def click_build_when_change_is_pushed_to_gitlab find('label', text: 'Build when a change is pushed to GitLab').find(:xpath, "..").find('input').click end diff --git a/qa/spec/page/base_spec.rb b/qa/spec/page/base_spec.rb index 52345876149..146e71da933 100644 --- a/qa/spec/page/base_spec.rb +++ b/qa/spec/page/base_spec.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# rubocop:disable QA/ElementWithPattern RSpec.describe QA::Page::Base do describe 'page helpers' do it 'exposes helpful page helpers' do @@ -11,12 +12,12 @@ RSpec.describe QA::Page::Base do subject do Class.new(described_class) do view 'path/to/some/view.html.haml' do - element :something, 'string pattern' # rubocop:disable QA/ElementWithPattern - element :something_else, /regexp pattern/ # rubocop:disable QA/ElementWithPattern + element :something, 'string pattern' + element :something_else, /regexp pattern/ end view 'path/to/some/_partial.html.haml' do - element :another_element, 'string pattern' # rubocop:disable QA/ElementWithPattern + element :another_element, 'string pattern' end end end @@ -95,6 +96,7 @@ RSpec.describe QA::Page::Base do describe '#all_elements' do before do allow(subject).to receive(:all) + allow(subject).to receive(:wait_for_requests) end it 'raises an error if count or minimum are not specified' do @@ -108,7 +110,7 @@ RSpec.describe QA::Page::Base do end end - context 'elements' do + describe 'elements' do subject do Class.new(described_class) do view 'path/to/some/view.html.haml' do @@ -133,35 +135,37 @@ RSpec.describe QA::Page::Base do describe '#visible?', 'Page is currently visible' do let(:page) { subject.new } + before do + allow(page).to receive(:wait_for_requests) + end + context 'with elements' do - context 'on the page' do - before do - # required elements not there, meaning not on page - allow(page).to receive(:has_no_element?).and_return(false) - end + before do + allow(page).to receive(:has_no_element?).and_return(has_no_element) + end + + context 'with element on the page' do + let(:has_no_element) { false } it 'is visible' do expect(page).to be_visible end - end - context 'not on the page' do - before do - # required elements are not on the page - allow(page).to receive(:has_no_element?).and_return(true) + it 'does not raise error if page has elements' do + expect { page.visible? }.not_to raise_error end + end + + context 'with element not on the page' do + let(:has_no_element) { true } it 'is not visible' do expect(page).not_to be_visible end end - - it 'does not raise error if page has elements' do - expect { page.visible? }.not_to raise_error - end end - context 'no elements' do + context 'with no elements' do subject do Class.new(described_class) do view 'path/to/some/view.html.haml' do @@ -180,3 +184,4 @@ RSpec.describe QA::Page::Base do end end end +# rubocop:enable QA/ElementWithPattern diff --git a/qa/spec/page/logging_spec.rb b/qa/spec/page/logging_spec.rb index 054332eea29..93a08108787 100644 --- a/qa/spec/page/logging_spec.rb +++ b/qa/spec/page/logging_spec.rb @@ -72,41 +72,47 @@ RSpec.describe QA::Support::Page::Logging do end it 'logs has_element?' do - expect { subject.has_element?(:element) } - .to output(/has_element\? :element \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/o).to_stdout_from_any_process + expect { subject.has_element?(:element) }.to output( + /has_element\? :element \(wait: #{Capybara.default_max_wait_time}\) returned: true/o + ).to_stdout_from_any_process end it 'logs has_element? with text' do - expect { subject.has_element?(:element, text: "some text") } - .to output(/has_element\? :element with text "some text" \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/o).to_stdout_from_any_process + expect { subject.has_element?(:element, text: "some text") }.to output( + /has_element\? :element with text "some text" \(wait: #{Capybara.default_max_wait_time}\) returned: true/o + ).to_stdout_from_any_process end it 'logs has_no_element?' do allow(page).to receive(:has_no_css?).and_return(true) - expect { subject.has_no_element?(:element) } - .to output(/has_no_element\? :element \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/o).to_stdout_from_any_process + expect { subject.has_no_element?(:element) }.to output( + /has_no_element\? :element \(wait: #{Capybara.default_max_wait_time}\) returned: true/o + ).to_stdout_from_any_process end it 'logs has_no_element? with text' do allow(page).to receive(:has_no_css?).and_return(true) - expect { subject.has_no_element?(:element, text: "more text") } - .to output(/has_no_element\? :element with text "more text" \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/o).to_stdout_from_any_process + expect { subject.has_no_element?(:element, text: "more text") }.to output( + /has_no_element\? :element with text "more text" \(wait: #{Capybara.default_max_wait_time}\) returned: true/o + ).to_stdout_from_any_process end it 'logs has_text?' do allow(page).to receive(:has_text?).and_return(true) - expect { subject.has_text? 'foo' } - .to output(/has_text\?\('foo', wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned true/o).to_stdout_from_any_process + expect { subject.has_text? 'foo' }.to output( + /has_text\?\('foo', wait: #{Capybara.default_max_wait_time}\) returned true/o + ).to_stdout_from_any_process end it 'logs has_no_text?' do allow(page).to receive(:has_no_text?).with('foo', any_args).and_return(true) - expect { subject.has_no_text? 'foo' } - .to output(/has_no_text\?\('foo', wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned true/o).to_stdout_from_any_process + expect { subject.has_no_text? 'foo' }.to output( + /has_no_text\?\('foo', wait: #{Capybara.default_max_wait_time}\) returned true/o + ).to_stdout_from_any_process end it 'logs finished_loading?' do @@ -123,7 +129,7 @@ RSpec.describe QA::Support::Page::Logging do .to output(/end within element :element/).to_stdout_from_any_process end - context 'all_elements' do + context 'with all_elements' do it 'logs the number of elements found' do allow(page).to receive(:all).and_return([1, 2]) diff --git a/qa/spec/resource/api_fabricator_spec.rb b/qa/spec/resource/api_fabricator_spec.rb index ec9907916eb..581236e5ac5 100644 --- a/qa/spec/resource/api_fabricator_spec.rb +++ b/qa/spec/resource/api_fabricator_spec.rb @@ -156,7 +156,7 @@ RSpec.describe QA::Resource::ApiFabricator do Fabrication of FooBarResource using the API failed (400) with `#{raw_post}`. Correlation Id: foobar Sentry Url: https://sentry.gitlab.net/gitlab/staginggitlabcom/?environment=gstg-cny&query=correlation_id%3A%22foobar%22 - Kibana Url: https://nonprod-log.gitlab.net/app/discover#/?_a=(query:(language:kuery,query:'json.correlation_id%20:%20foobar')) + Kibana Url: https://nonprod-log.gitlab.net/app/discover#/?_a=(query:(language:kuery,query:'json.correlation_id%20:%20foobar'))&_g=(time:(from:now-24h%2Fh,to:now)) ERROR end end diff --git a/qa/spec/resource/base_spec.rb b/qa/spec/resource/base_spec.rb index eab205ec5d1..6dac8e0e3ee 100644 --- a/qa/spec/resource/base_spec.rb +++ b/qa/spec/resource/base_spec.rb @@ -4,6 +4,7 @@ RSpec.describe QA::Resource::Base do include QA::Support::Helpers::StubEnv let(:resource) { spy('resource') } + let(:api_client) { instance_double('Runtime::API::Client') } let(:location) { 'http://location' } let(:log_regex) { %r{==> Built a MyResource with username 'qa' via #{method} in [\d.\-e]+ seconds+} } @@ -114,6 +115,7 @@ RSpec.describe QA::Resource::Base do allow(QA::Runtime::Logger).to receive(:debug) allow(resource).to receive(:api_support?).and_return(true) allow(resource).to receive(:fabricate_via_api!) + allow(resource).to receive(:api_client) { api_client } end it 'logs the resource and build method' do @@ -154,7 +156,6 @@ RSpec.describe QA::Resource::Base do before do allow(QA::Runtime::Logger).to receive(:debug) - # allow(resource).to receive(:fabricate!) end it 'logs the resource and build method' do diff --git a/qa/spec/runtime/script_extensions/interceptor_spec.rb b/qa/spec/runtime/script_extensions/interceptor_spec.rb new file mode 100644 index 00000000000..28e8007973c --- /dev/null +++ b/qa/spec/runtime/script_extensions/interceptor_spec.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +RSpec.describe 'Interceptor' do + let(:browser) { Capybara.current_session } + # need a real host for the js runtime + let(:url) { "file://#{__dir__}/../../../qa/fixtures/script_extensions/test.html" } + + before(:context) do + skip 'Only can test for chrome' unless QA::Runtime::Env.can_intercept? + + QA::Runtime::Browser.configure! + QA::Runtime::Browser::Session.enable_interception + end + + after(:context) do + QA::Runtime::Browser::Session.disable_interception + end + + before do + browser.visit url + + clear_cache + end + + after do + browser.visit 'about:blank' + end + + context 'with Interceptor' do + context 'with caching' do + it 'checks the cache' do + expect(check_cache).to be(true) + end + + it 'returns false if the cache cannot be accessed' do + browser.visit 'about:blank' + + expect(check_cache).to be(false) + end + + it 'gets and sets the cache data' do + commit_to_cache({ foo: 'bar' }) + + expect(get_cache['data']).to eql({ 'foo' => 'bar' }) + end + end + + context 'when intercepting' do + let(:resource_url) { 'chrome://chrome-urls' } + + it 'intercepts fetch errors' do + trigger_fetch(resource_url, 'GET') + + errors = get_cache['errors'] + + expect(errors.size).to be(1) + expect(errors[0]['status']).to be(-1) + expect(errors[0]['method']).to eql('GET') + expect(errors[0]['url']).to eql(resource_url) + end + + it 'intercepts xhr' do + trigger_xhr(resource_url, 'POST') + + errors = get_cache['errors'] + + expect(errors.size).to be(1) + expect(errors[0]['status']).to be(-1) + expect(errors[0]['method']).to eql('POST') + expect(errors[0]['url']).to eql(resource_url) + end + end + end + + def clear_cache + browser.execute_script <<~JS + Interceptor.saveCache({}) + JS + end + + def check_cache + browser.execute_script <<~JS + return Interceptor.checkCache() + JS + end + + def trigger_fetch(url, method) + browser.execute_script <<~JS + (() => { + fetch('#{url}', { method: '#{method}' }) + })() + JS + end + + def trigger_xhr(url, method) + browser.execute_script <<~JS + (() => { + let xhr = new XMLHttpRequest(); + xhr.open('#{method}', '#{url}') + xhr.send() + })() + JS + end + + def commit_to_cache(payload) + browser.execute_script <<~JS + Interceptor.commitToCache((cache) => { + cache.data = JSON.parse('#{payload.to_json}'); + return cache + }) + JS + end + + def get_cache + browser.execute_script <<~JS + return Interceptor.getCache() + JS + end +end diff --git a/qa/spec/service/docker_run/gitlab_runner_spec.rb b/qa/spec/service/docker_run/gitlab_runner_spec.rb index a8838db10cf..d9f201cf67e 100644 --- a/qa/spec/service/docker_run/gitlab_runner_spec.rb +++ b/qa/spec/service/docker_run/gitlab_runner_spec.rb @@ -24,6 +24,7 @@ module QA before do allow(subject).to receive(:shell) + allow(subject).to receive(:wait_until_running_and_configured) end context 'defaults' do diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb index 655b0088feb..b81c41bb79c 100644 --- a/qa/spec/spec_helper.rb +++ b/qa/spec/spec_helper.rb @@ -20,6 +20,7 @@ RSpec.configure do |config| config.add_formatter QA::Support::Formatters::ContextFormatter config.add_formatter QA::Support::Formatters::QuarantineFormatter + config.add_formatter QA::Support::Formatters::FeatureFlagFormatter config.add_formatter QA::Support::Formatters::TestStatsFormatter if QA::Runtime::Env.export_metrics? config.before(:suite) do |suite| @@ -77,7 +78,17 @@ RSpec.configure do |config| # If any tests failed, leave the resources behind to help troubleshoot, otherwise remove them. # Do not remove the shared resource on live environments - QA::Resource::ReusableCollection.remove_all_via_api! if !suite.reporter.failed_examples.present? && !QA::Runtime::Env.running_on_dot_com? + begin + next if suite.reporter.failed_examples.present? + next unless QA::Runtime::Scenario.attributes.include?(:gitlab_address) + next if QA::Runtime::Env.running_on_dot_com? + + QA::Resource::ReusableCollection.remove_all_via_api! + rescue QA::Resource::Errors::InternalServerError => e + # Temporarily prevent this error from failing jobs while the cause is investigated + # See https://gitlab.com/gitlab-org/gitlab/-/issues/354387 + QA::Runtime::Logger.debug(e.message) + end end config.append_after(:suite) do @@ -105,6 +116,9 @@ RSpec.configure do |config| # show exception that triggers a retry if verbose_retry is set to true config.display_try_failure_messages = true + # This option allows to use shorthand aliases for adding :focus metadata - fit, fdescribe and fcontext + config.filter_run_when_matching :focus + if ENV['CI'] && !QA::Runtime::Env.disable_rspec_retry? non_quarantine_retries = QA::Runtime::Env.ci_project_name =~ /staging|canary|production/ ? 3 : 2 config.around do |example| diff --git a/qa/spec/specs/allure_report_spec.rb b/qa/spec/specs/allure_report_spec.rb index 86ceaf51cbb..85befb2f602 100644 --- a/qa/spec/specs/allure_report_spec.rb +++ b/qa/spec/specs/allure_report_spec.rb @@ -3,7 +3,7 @@ describe QA::Runtime::AllureReport do include QA::Support::Helpers::StubEnv - let(:rspec_config) { double('RSpec::Core::Configuration', 'add_formatter': nil, append_after: nil) } + let(:rspec_config) { instance_double('RSpec::Core::Configuration', 'add_formatter': nil, append_after: nil) } let(:png_path) { 'png_path' } let(:html_path) { 'html_path' } @@ -42,11 +42,14 @@ describe QA::Runtime::AllureReport do context 'with report generation enabled' do let(:generate_report) { 'true' } + let(:session) { instance_double('Capybara::Session') } + let(:attributes) { class_spy('Runtime::Scenario') } + let(:version_response) { instance_double('HTTPResponse', code: 200, body: versions.to_json) } + let(:png_file) { 'png-file' } let(:html_file) { 'html-file' } let(:ci_job) { 'ee:relative 5' } let(:versions) { { version: '14', revision: '6ced31db947' } } - let(:session) { double('session') } let(:browser_log) { ['log message 1', 'log message 2'] } before do @@ -54,11 +57,13 @@ describe QA::Runtime::AllureReport do stub_env('CI_JOB_NAME', ci_job) stub_env('GITLAB_QA_ADMIN_ACCESS_TOKEN', 'token') + stub_const('QA::Runtime::Scenario', attributes) + allow(Allure).to receive(:add_attachment) allow(File).to receive(:open).with(png_path) { png_file } allow(File).to receive(:open).with(html_path) { html_file } - allow(RestClient::Request).to receive(:execute) { double('response', code: 200, body: versions.to_json) } - allow(QA::Runtime::Scenario).to receive(:method_missing).with(:gitlab_address).and_return('gitlab.com') + allow(RestClient::Request).to receive(:execute) { version_response } + allow(attributes).to receive(:gitlab_address).and_return("https://gitlab.com") allow(Capybara).to receive(:current_session).and_return(session) allow(session).to receive_message_chain('driver.browser.logs.get').and_return(browser_log) diff --git a/qa/spec/specs/helpers/feature_flag_spec.rb b/qa/spec/specs/helpers/feature_flag_spec.rb new file mode 100644 index 00000000000..a1300ecf073 --- /dev/null +++ b/qa/spec/specs/helpers/feature_flag_spec.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +require 'rspec/core/sandbox' + +RSpec.describe QA::Specs::Helpers::FeatureFlag do + include QA::Support::Helpers::StubEnv + include QA::Specs::Helpers::RSpec + + around do |ex| + RSpec::Core::Sandbox.sandboxed do |config| + config.add_formatter QA::Support::Formatters::ContextFormatter + config.add_formatter QA::Support::Formatters::QuarantineFormatter + config.add_formatter QA::Support::Formatters::FeatureFlagFormatter + + # If there is an example-within-an-example, we want to make sure the inner example + # does not get a reference to the outer example (the real spec) if it calls + # something like `pending` + config.before(:context) { RSpec.current_example = nil } + + config.color_mode = :off + + ex.run + end + end + + describe '.skip_or_run_feature_flag_tests_or_contexts' do + shared_examples 'runs with given feature flag metadata' do |metadata| + it do + group = describe_successfully 'Feature flag test', feature_flag: metadata do + it('passes') {} + end + + expect(group.examples.first.execution_result.status).to eq(:passed) + end + end + + shared_examples 'skips with given feature flag metadata' do |metadata| + it do + group = describe_successfully 'Feature flag test', feature_flag: metadata do + it('is skipped') {} + end + + expect(group.examples.first.execution_result.status).to eq(:pending) + end + end + + context 'when run on staging' do + before(:context) do + QA::Runtime::Scenario.define(:gitlab_address, 'https://staging.gitlab.com') + end + + context 'when no scope is defined' do + it_behaves_like 'runs with given feature flag metadata', { name: 'no_scope_ff' } + + it 'is skipped if quarantine tag is also applied' do + group = describe_successfully( + 'Feature flag with no scope', + feature_flag: { name: 'quarantine_with_ff' }, + quarantine: { + issue: 'https://gitlab.com/test-group/test/-/issues/123', + type: 'bug' + } + ) do + it('is skipped') {} + end + + expect(group.examples.first.execution_result.status).to eq(:pending) + end + end + + it_behaves_like 'runs with given feature flag metadata', { name: 'actor_ff', scope: :project } + + it_behaves_like 'skips with given feature flag metadata', { name: 'global_ff', scope: :global } + + context 'when should be skipped in a specific job' do + before do + stub_env('CI_JOB_NAME', 'job-to-skip') + end + + it 'is skipped for that job' do + group = describe_successfully( + 'Test should be skipped', + feature_flag: { name: 'skip_job_ff' }, + except: { job: 'job-to-skip' } + ) do + it('does not run on staging in specified job') {} + end + + expect(group.examples.first.execution_result.status).to eq(:pending) + end + end + + context 'when should only run in a specific job' do + before do + stub_env('CI_JOB_NAME', 'job-to-run') + end + + it 'is run for that job' do + group = describe_successfully( + 'Test should run', + feature_flag: { name: 'run_job_ff' }, + only: { job: 'job-to-run' } + ) do + it('runs on staging in specified job') {} + end + + expect(group.examples.first.execution_result.status).to eq(:passed) + end + + it 'skips if test is set to only run in a job different from current CI job' do + group = describe_successfully( + 'Test should be skipped', + feature_flag: { name: 'skip_job_ff' }, + only: { job: 'other-job' } + ) do + it('does not run on staging in specified job') {} + end + + expect(group.examples.first.execution_result.status).to eq(:pending) + end + end + end + + context 'when run on production' do + before(:context) do + QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.com') + end + + context 'when no scope is defined' do + it_behaves_like 'skips with given feature flag metadata', { name: 'no_scope_ff' } + + context 'for only one test in the example group' do + it 'only skips specified test and runs all others' do + group = describe_successfully 'Feature flag set for one test' do + it('is skipped', feature_flag: { name: 'single_test_ff' }) {} + it('passes') {} + end + + expect(group.examples[0].execution_result.status).to eq(:pending) + expect(group.examples[1].execution_result.status).to eq(:passed) + end + end + end + + it_behaves_like 'skips with given feature flag metadata', { name: 'actor_ff', scope: :project } + + it_behaves_like 'skips with given feature flag metadata', { name: 'global_ff', scope: :global } + end + + # The nightly package job, for example, does not run against a live environment with + # a defined gitlab_address. In this case, feature_flag tag logic can be safely ignored + context 'when run without a gitlab address specified' do + before(:context) do + QA::Runtime::Scenario.define(:gitlab_address, nil) + end + + it_behaves_like 'runs with given feature flag metadata', { name: 'no_scope_ff' } + + it_behaves_like 'runs with given feature flag metadata', { name: 'actor_ff', scope: :project } + + it_behaves_like 'runs with given feature flag metadata', { name: 'global_ff', scope: :global } + end + end +end diff --git a/qa/spec/specs/runner_spec.rb b/qa/spec/specs/runner_spec.rb index e52ca1fb17c..d5e442acfe7 100644 --- a/qa/spec/specs/runner_spec.rb +++ b/qa/spec/specs/runner_spec.rb @@ -86,6 +86,41 @@ RSpec.describe QA::Specs::Runner do end end + context 'when test_metadata_only is set as an option' do + let(:rspec_config) { instance_double('RSpec::Core::Configuration') } + let(:output_file) { Pathname.new('/root/tmp/test-metadata.json') } + + before do + QA::Runtime::Scenario.define(:test_metadata_only, true) + allow(RSpec).to receive(:configure).and_yield(rspec_config) + allow(rspec_config).to receive(:add_formatter) + allow(rspec_config).to receive(:fail_if_no_examples=) + end + + it 'sets the `--dry-run` flag' do + expect_rspec_runner_arguments(['--dry-run', '--tag', '~orchestrated', '--tag', '~transient', '--tag', '~geo', *described_class::DEFAULT_TEST_PATH_ARGS], [$stderr, anything]) + + subject.perform + end + + it 'configures json formatted output to file' do + allow(QA::Runtime::Path).to receive(:qa_root).and_return('/root') + + expect(rspec_config).to receive(:add_formatter) + .with(QA::Support::JsonFormatter, output_file) + expect(rspec_config).to receive(:fail_if_no_examples=) + .with(true) + + allow(RSpec::Core::Runner).to receive(:run).and_return(0) + + subject.perform + end + + after do + QA::Runtime::Scenario.attributes.delete(:test_metadata_only) + end + end + context 'when tags are set' do subject { described_class.new.tap { |runner| runner.tags = %i[orchestrated github] } } diff --git a/qa/spec/support/shared_examples/scenario_shared_examples.rb b/qa/spec/specs/scenario_shared_examples.rb index 5e448349cf9..7d806d50d21 100644 --- a/qa/spec/support/shared_examples/scenario_shared_examples.rb +++ b/qa/spec/specs/scenario_shared_examples.rb @@ -2,10 +2,10 @@ module QA RSpec.shared_examples 'a QA scenario class' do - let(:attributes) { spy('Runtime::Scenario') } - let(:runner) { spy('Specs::Runner') } - let(:release) { spy('Runtime::Release') } - let(:feature) { spy('Runtime::Feature') } + let(:attributes) { class_spy('Runtime::Scenario') } + let(:runner) { class_spy('Specs::Runner') } + let(:release) { class_spy('Runtime::Release') } + let(:feature) { class_spy('Runtime::Feature') } let(:args) { { gitlab_address: 'http://gitlab_address' } } let(:named_options) { %w[--address http://gitlab_address] } @@ -45,7 +45,7 @@ module QA expect(runner).to have_received(:tags=).with(tags) end - context 'specifying RSpec options' do + context 'with RSpec options' do it 'sets options on runner' do subject.perform(args, *options) diff --git a/qa/spec/specs/spec_helper.rb b/qa/spec/specs/spec_helper.rb new file mode 100644 index 00000000000..e4514c6c64f --- /dev/null +++ b/qa/spec/specs/spec_helper.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative '../../qa' + +require_relative 'scenario_shared_examples' diff --git a/qa/spec/support/formatters/test_stats_formatter_spec.rb b/qa/spec/support/formatters/test_stats_formatter_spec.rb index 518c7407ba6..ba59588d186 100644 --- a/qa/spec/support/formatters/test_stats_formatter_spec.rb +++ b/qa/spec/support/formatters/test_stats_formatter_spec.rb @@ -8,14 +8,15 @@ describe QA::Support::Formatters::TestStatsFormatter do include QA::Specs::Helpers::RSpec include ActiveSupport::Testing::TimeHelpers - let(:url) { "http://influxdb.net" } - let(:token) { "token" } - let(:ci_timestamp) { "2021-02-23T20:58:41Z" } - let(:ci_job_name) { "test-job 1/5" } - let(:ci_job_url) { "url" } - let(:ci_pipeline_url) { "url" } - let(:ci_pipeline_id) { "123" } + let(:url) { 'http://influxdb.net' } + let(:token) { 'token' } + let(:ci_timestamp) { '2021-02-23T20:58:41Z' } + let(:ci_job_name) { 'test-job 1/5' } + let(:ci_job_url) { 'url' } + let(:ci_pipeline_url) { 'url' } + let(:ci_pipeline_id) { '123' } let(:run_type) { 'staging-full' } + let(:smoke) { 'false' } let(:reliable) { 'false' } let(:quarantined) { 'false' } let(:influx_client) { instance_double('InfluxDB2::Client', create_write_api: influx_write_api) } @@ -25,6 +26,7 @@ describe QA::Support::Formatters::TestStatsFormatter do let(:ui_fabrication) { 0 } let(:api_fabrication) { 0 } let(:fabrication_resources) { {} } + let(:testcase) { 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234' } let(:influx_client_args) do { @@ -42,14 +44,15 @@ describe QA::Support::Formatters::TestStatsFormatter do name: 'stats export spec', file_path: file_path.gsub('./qa/specs/features', ''), status: :passed, + smoke: smoke, reliable: reliable, quarantined: quarantined, - retried: "false", - job_name: "test-job", - merge_request: "false", + retried: 'false', + job_name: 'test-job', + merge_request: 'false', run_type: run_type, stage: stage.match(%r{\d{1,2}_(\w+)}).captures.first, - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234' + testcase: testcase }, fields: { id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]', @@ -78,12 +81,6 @@ describe QA::Support::Formatters::TestStatsFormatter do around do |example| RSpec::Core::Sandbox.sandboxed do |config| config.formatter = QA::Support::Formatters::TestStatsFormatter - - config.append_after do |example| - example.metadata[:api_fabrication] = Thread.current[:api_fabrication] - example.metadata[:browser_ui_fabrication] = Thread.current[:browser_ui_fabrication] - end - config.before(:context) { RSpec.current_example = nil } example.run @@ -93,10 +90,11 @@ describe QA::Support::Formatters::TestStatsFormatter do before do allow(InfluxDB2::Client).to receive(:new).with(url, token, **influx_client_args) { influx_client } allow(QA::Tools::TestResourceDataProcessor).to receive(:resources) { fabrication_resources } + allow_any_instance_of(RSpec::Core::Example::ExecutionResult).to receive(:run_time).and_return(0) # rubocop:disable RSpec/AnyInstanceOf end - context "without influxdb variables configured" do - it "skips export without influxdb url" do + context 'without influxdb variables configured' do + it 'skips export without influxdb url' do stub_env('QA_INFLUXDB_URL', nil) stub_env('QA_INFLUXDB_TOKEN', nil) @@ -105,7 +103,7 @@ describe QA::Support::Formatters::TestStatsFormatter do expect(influx_client).not_to have_received(:create_write_api) end - it "skips export without influxdb token" do + it 'skips export without influxdb token' do stub_env('QA_INFLUXDB_URL', url) stub_env('QA_INFLUXDB_TOKEN', nil) @@ -145,6 +143,19 @@ describe QA::Support::Formatters::TestStatsFormatter do end end + context 'with smoke spec' do + let(:smoke) { 'true' } + + it 'exports data to influxdb with correct smoke tag' do + run_spec do + it('spec', :smoke, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {} + end + + expect(influx_write_api).to have_received(:write).once + expect(influx_write_api).to have_received(:write).with(data: [data]) + end + end + context 'with quarantined spec' do let(:quarantined) { 'true' } @@ -210,16 +221,18 @@ describe QA::Support::Formatters::TestStatsFormatter do end context 'with fabrication runtimes' do - let(:ui_fabrication) { 10 } let(:api_fabrication) { 4 } - - before do - Thread.current[:api_fabrication] = api_fabrication - Thread.current[:browser_ui_fabrication] = ui_fabrication - end + let(:ui_fabrication) { 10 } + let(:testcase) { nil } it 'exports data to influxdb with fabrication times' do - run_spec + run_spec do + # Main logic tracks fabrication time in thread local variable and injects it as metadata from + # global after hook defined in main spec_helper. + # + # Inject the values directly since we do not load e2e test spec_helper in unit tests + it('spec', api_fabrication: 4, browser_ui_fabrication: 10) {} + end expect(influx_write_api).to have_received(:write).once expect(influx_write_api).to have_received(:write).with(data: [data]) diff --git a/qa/spec/support/loglinking_spec.rb b/qa/spec/support/loglinking_spec.rb index cba8a139767..e02ae45ee93 100644 --- a/qa/spec/support/loglinking_spec.rb +++ b/qa/spec/support/loglinking_spec.rb @@ -28,7 +28,7 @@ RSpec.describe QA::Support::Loglinking do expect(QA::Support::Loglinking.failure_metadata('foo123')).to eql(<<~ERROR.chomp) Correlation Id: foo123 - Kibana Url: https://kibana.address/app/discover#/?_a=(query:(language:kuery,query:'json.correlation_id%20:%20foo123')) + Kibana Url: https://kibana.address/app/discover#/?_a=(query:(language:kuery,query:'json.correlation_id%20:%20foo123'))&_g=(time:(from:now-24h%2Fh,to:now)) ERROR end end @@ -83,7 +83,7 @@ RSpec.describe QA::Support::Loglinking do describe '.logging_environment' do let(:staging_address) { 'https://staging.gitlab.com' } let(:staging_ref_address) { 'https://staging-ref.gitlab.com' } - let(:production_address) { 'https://www.gitlab.com' } + let(:production_address) { 'https://gitlab.com' } let(:pre_prod_address) { 'https://pre.gitlab.com' } let(:logging_env_array) do [ diff --git a/qa/spec/support/page_error_checker_spec.rb b/qa/spec/support/page_error_checker_spec.rb index b9b085fa7b9..7c8aaeb182a 100644 --- a/qa/spec/support/page_error_checker_spec.rb +++ b/qa/spec/support/page_error_checker_spec.rb @@ -238,6 +238,88 @@ RSpec.describe QA::Support::PageErrorChecker do end end + describe '::log_request_errors' do + let(:page_url) { 'https://baz.foo' } + let(:browser) { double('browser', current_url: page_url) } + let(:driver) { double('driver', browser: browser) } + let(:session) { double('session', driver: driver) } + + before do + allow(Capybara).to receive(:current_session).and_return(session) + end + + it 'logs from the error cache' do + error = { + 'url' => 'https://foo.bar', + 'status' => 500, + 'method' => 'GET', + 'headers' => { 'x-request-id' => '12345' } + } + + expect(page).to receive(:driver).and_return(driver) + expect(page).to receive(:execute_script).and_return({ 'errors' => [error] }) + expect(page).to receive(:execute_script) + + expect(QA::Runtime::Logger).to receive(:debug).with("Fetching API error cache for #{page_url}") + expect(QA::Runtime::Logger).to receive(:error).with(<<~ERROR.chomp) + Interceptor Api Errors + [500] GET https://foo.bar -- Correlation Id: 12345 + ERROR + + QA::Support::PageErrorChecker.log_request_errors(page) + end + + it 'removes duplicates' do + error = { + 'url' => 'https://foo.bar', + 'status' => 500, + 'method' => 'GET', + 'headers' => { 'x-request-id' => '12345' } + } + expect(page).to receive(:driver).and_return(driver) + expect(page).to receive(:execute_script).and_return({ 'errors' => [error, error, error] }) + expect(page).to receive(:execute_script) + + expect(QA::Runtime::Logger).to receive(:debug).with("Fetching API error cache for #{page_url}") + expect(QA::Runtime::Logger).to receive(:error).with(<<~ERROR.chomp).exactly(1).time + Interceptor Api Errors + [500] GET https://foo.bar -- Correlation Id: 12345 + ERROR + + QA::Support::PageErrorChecker.log_request_errors(page) + end + + it 'chops the url query string' do + error = { + 'url' => 'https://foo.bar?query={ sensitive-data: 12345 }', + 'status' => 500, + 'method' => 'GET', + 'headers' => { 'x-request-id' => '12345' } + } + expect(page).to receive(:driver).and_return(driver) + expect(page).to receive(:execute_script).and_return({ 'errors' => [error] }) + expect(page).to receive(:execute_script) + + expect(QA::Runtime::Logger).to receive(:debug).with("Fetching API error cache for #{page_url}") + expect(QA::Runtime::Logger).to receive(:error).with(<<~ERROR.chomp) + Interceptor Api Errors + [500] GET https://foo.bar -- Correlation Id: 12345 + ERROR + + QA::Support::PageErrorChecker.log_request_errors(page) + end + + it 'returns if cache is nil' do + expect(page).to receive(:driver).and_return(driver) + expect(page).to receive(:execute_script).and_return(nil) + + expect(QA::Runtime::Logger).to receive(:debug).with("Fetching API error cache for #{page_url}") + expect(QA::Runtime::Logger).not_to receive(:error) + + QA::Support::PageErrorChecker.log_request_errors(page) + end + end + describe '.logs' do before do logs_class = Class.new do diff --git a/qa/spec/support/shared_contexts/merge_train_spec_with_user_prep.rb b/qa/spec/support/shared_contexts/merge_train_spec_with_user_prep.rb new file mode 100644 index 00000000000..9d1a37cb0b8 --- /dev/null +++ b/qa/spec/support/shared_contexts/merge_train_spec_with_user_prep.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module QA + RSpec.shared_context 'merge train spec with user prep' do + let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" } + let(:file_name) { Faker::Lorem.word } + let(:mr_title) { Faker::Lorem.sentence } + let(:admin_api_client) { Runtime::API::Client.as_admin } + + let(:user) do + Resource::User.fabricate_via_api! do |user| + user.api_client = admin_api_client + end + end + + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'pipeline-for-merge-trains' + end + end + + let!(:runner) do + Resource::Runner.fabricate! do |runner| + runner.project = project + runner.name = executor + runner.tags = [executor] + end + end + + let!(:project_files) do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files( + [ + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + test_merge_train: + tags: + - #{executor} + script: + - sleep 10 + - echo 'OK!' + only: + - merge_requests + YAML + }, + { + file_path: file_name, + content: Faker::Lorem.sentence + } + ] + ) + end + end + + before do + project.add_member(user, Resource::Members::AccessLevel::MAINTAINER) + + Flow::Login.sign_in + project.visit! + Flow::MergeRequest.enable_merge_trains + + Flow::Login.sign_in(as: user) + + Resource::MergeRequest.fabricate_via_api! do |merge_request| + merge_request.title = mr_title + merge_request.project = project + merge_request.description = Faker::Lorem.sentence + merge_request.target_new_branch = false + merge_request.update_existing_file = true + merge_request.file_name = file_name + merge_request.file_content = Faker::Lorem.sentence + end.visit! + + Page::MergeRequest::Show.perform do |show| + show.has_pipeline_status?('passed') + show.try_to_merge! + end + end + + after do + runner&.remove_via_api! + user&.remove_via_api! + end + end +end diff --git a/qa/spec/support/shared_contexts/variable_inheritance_shared_context.rb b/qa/spec/support/shared_contexts/variable_inheritance_shared_context.rb new file mode 100644 index 00000000000..1dc8870d4d9 --- /dev/null +++ b/qa/spec/support/shared_contexts/variable_inheritance_shared_context.rb @@ -0,0 +1,150 @@ +# frozen_string_literal: true + +module QA + RSpec.shared_context 'variable inheritance test prep' do + let(:random_string) { Faker::Alphanumeric.alphanumeric(number: 8) } + + let(:group) do + Resource::Group.fabricate_via_api! do |group| + group.path = "group-for-variable-inheritance-#{random_string}" + end + end + + let(:upstream_project) do + Resource::Project.fabricate_via_api! do |project| + project.group = group + project.name = 'upstream-variable-inheritance' + project.description = 'Project for pipeline with variable defined via UI - Upstream' + end + end + + let(:downstream1_project) do + Resource::Project.fabricate_via_api! do |project| + project.group = group + project.name = 'downstream1-variable-inheritance' + project.description = 'Project for pipeline with variable defined via UI - Downstream' + end + end + + let(:downstream2_project) do + Resource::Project.fabricate_via_api! do |project| + project.group = group + project.name = 'downstream2-variable-inheritance' + project.description = 'Project for pipeline with variable defined via UI - Downstream' + end + end + + let!(:runner) do + Resource::Runner.fabricate! do |runner| + runner.token = group.reload!.runners_token + runner.name = random_string + runner.tags = [random_string] + end + end + + before do + Runtime::Feature.enable(:ci_trigger_forward_variables) + Flow::Login.sign_in + end + + after do + runner.remove_via_api! + Runtime::Feature.disable(:ci_trigger_forward_variables) + end + + def start_pipeline_with_variable + upstream_project.visit! + Flow::Pipeline.wait_for_latest_pipeline + Page::Project::Pipeline::Index.perform(&:click_run_pipeline_button) + Page::Project::Pipeline::New.perform do |new| + new.add_variable('TEST_VAR', 'This is great!') + new.click_run_pipeline_button + end + end + + def add_ci_file(project, files) + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add CI config file' + commit.add_files(files) + end + end + + def visit_job_page(pipeline_title, job_name) + Page::Project::Pipeline::Show.perform do |show| + show.expand_child_pipeline(title: pipeline_title) + show.click_job(job_name) + end + end + + def verify_job_log_shows_variable_value + Page::Project::Job::Show.perform do |show| + show.wait_until { show.successful? } + expect(show.output).to have_content('This is great!') + end + end + + def verify_job_log_does_not_show_variable_value + Page::Project::Job::Show.perform do |show| + show.wait_until { show.successful? } + expect(show.output).to have_no_content('This is great!') + end + end + + def upstream_child1_ci_file + { + file_path: '.child1-ci.yml', + content: <<~YAML + child1_job: + stage: test + tags: ["#{random_string}"] + script: + - echo $TEST_VAR + - echo Done! + YAML + } + end + + def upstream_child2_ci_file + { + file_path: '.child2-ci.yml', + content: <<~YAML + child2_job: + stage: test + tags: ["#{random_string}"] + script: + - echo $TEST_VAR + - echo Done! + YAML + } + end + + def downstream1_ci_file + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + downstream1_job: + stage: deploy + tags: ["#{random_string}"] + script: + - echo $TEST_VAR + - echo Done! + YAML + } + end + + def downstream2_ci_file + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + downstream2_job: + stage: deploy + tags: ["#{random_string}"] + script: + - echo $TEST_VAR + - echo Done! + YAML + } + end + end +end diff --git a/qa/spec/support/wait_for_requests_spec.rb b/qa/spec/support/wait_for_requests_spec.rb index 2492820b67f..221d61ea2b4 100644 --- a/qa/spec/support/wait_for_requests_spec.rb +++ b/qa/spec/support/wait_for_requests_spec.rb @@ -5,37 +5,38 @@ RSpec.describe QA::Support::WaitForRequests do before do allow(subject).to receive(:finished_all_ajax_requests?).and_return(true) allow(subject).to receive(:finished_loading?).and_return(true) + allow(QA::Support::PageErrorChecker).to receive(:check_page_for_error_code) end context 'when skip_finished_loading_check is defaulted to false' do it 'calls finished_loading?' do - expect(subject).to receive(:finished_loading?).with(hash_including(wait: 1)) - subject.wait_for_requests + + expect(subject).to have_received(:finished_loading?).with(hash_including(wait: 1)) end end context 'when skip_finished_loading_check is true' do it 'does not call finished_loading?' do - expect(subject).not_to receive(:finished_loading?) - subject.wait_for_requests(skip_finished_loading_check: true) + + expect(subject).not_to have_received(:finished_loading?) end end context 'when skip_resp_code_check is defaulted to false' do it 'call report' do - allow(QA::Support::PageErrorChecker).to receive(:check_page_for_error_code).with(Capybara.page) - subject.wait_for_requests + + expect(QA::Support::PageErrorChecker).to have_received(:check_page_for_error_code).with(Capybara.page) end end context 'when skip_resp_code_check is true' do it 'does not parse for an error code' do - expect(QA::Support::PageErrorChecker).not_to receive(:check_page_for_error_code) - subject.wait_for_requests(skip_resp_code_check: true) + + expect(QA::Support::PageErrorChecker).not_to have_received(:check_page_for_error_code) end end end diff --git a/qa/spec/tools/reliable_report_spec.rb b/qa/spec/tools/reliable_report_spec.rb index 318b0833f62..d5c898779b8 100644 --- a/qa/spec/tools/reliable_report_spec.rb +++ b/qa/spec/tools/reliable_report_spec.rb @@ -5,7 +5,6 @@ describe QA::Tools::ReliableReport do subject(:run) { described_class.run(range: range, report_in_issue_and_slack: create_issue) } - let(:gitlab_response) { instance_double("RestClient::Response", code: 200, body: { web_url: issue_url }.to_json) } let(:slack_notifier) { instance_double("Slack::Notifier", post: nil) } let(:influx_client) { instance_double("InfluxDB2::Client", create_query_api: query_api) } let(:query_api) { instance_double("InfluxDB2::QueryApi") } @@ -71,6 +70,7 @@ describe QA::Tools::ReliableReport do |> filter(fn: (r) => r.status != "pending" and r.merge_request == "false" and r.quarantined == "false" and + r.smoke == "false" and r.reliable == "#{reliable}" and r._field == "id" ) @@ -117,7 +117,7 @@ describe QA::Tools::ReliableReport do stub_env("CI_API_V4_URL", "gitlab_api_url") stub_env("GITLAB_ACCESS_TOKEN", "gitlab_token") - allow(RestClient::Request).to receive(:execute).and_return(gitlab_response) + allow(RestClient::Request).to receive(:execute) allow(Slack::Notifier).to receive(:new).and_return(slack_notifier) allow(InfluxDB2::Client).to receive(:new).and_return(influx_client) @@ -138,6 +138,37 @@ describe QA::Tools::ReliableReport do context "with report creation" do let(:create_issue) { "true" } + let(:iid) { 2 } + let(:old_iid) { 1 } + let(:issue_endpoint) { "gitlab_api_url/projects/278964/issues" } + + let(:common_api_args) do + { + verify_ssl: false, + headers: { "PRIVATE-TOKEN" => "gitlab_token" } + } + end + + let(:create_issue_response) do + instance_double( + "RestClient::Response", + code: 200, + body: { web_url: issue_url, iid: iid }.to_json + ) + end + + let(:open_issues_response) do + instance_double( + "RestClient::Response", + code: 200, + body: [{ web_url: issue_url, iid: iid }, { web_url: issue_url, iid: old_iid }].to_json + ) + end + + let(:success_response) do + instance_double("RestClient::Response", code: 200, body: {}.to_json) + end + let(:issue_body) do <<~TXT.strip [[_TOC_]] @@ -156,19 +187,48 @@ describe QA::Tools::ReliableReport do TXT end - it "creates report issue", :aggregate_failures do + before do + allow(RestClient::Request).to receive(:execute).exactly(4).times.and_return( + create_issue_response, + open_issues_response, + success_response, + success_response + ) + end + + it "creates report issue" do expect { run }.to output.to_stdout expect(RestClient::Request).to have_received(:execute).with( method: :post, - url: "gitlab_api_url/projects/278964/issues", - verify_ssl: false, - headers: { "PRIVATE-TOKEN" => "gitlab_token" }, + url: issue_endpoint, payload: { title: "Reliable e2e test report", description: issue_body, - labels: "Quality,test,type::maintenance,reliable test report,automation:ml" - } + labels: "reliable test report,Quality,test,type::maintenance,automation:ml" + }, + **common_api_args + ) + expect(RestClient::Request).to have_received(:execute).with( + method: :get, + url: "#{issue_endpoint}?labels=reliable test report&state=opened", + **common_api_args + ) + expect(RestClient::Request).to have_received(:execute).with( + method: :put, + url: "#{issue_endpoint}/#{old_iid}", + payload: { + state_event: "close" + }, + **common_api_args + ) + expect(RestClient::Request).to have_received(:execute).with( + method: :post, + url: "#{issue_endpoint}/#{old_iid}/notes", + payload: { + body: "Closed issue in favor of ##{iid}" + }, + **common_api_args ) expect(slack_notifier).to have_received(:post).with( icon_emoji: ":tanuki-protect:", |