diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-20 09:16:11 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-20 09:16:11 +0000 |
| commit | edaa33dee2ff2f7ea3fac488d41558eb5f86d68c (patch) | |
| tree | 11f143effbfeba52329fb7afbd05e6e2a3790241 /qa | |
| parent | d8a5691316400a0f7ec4f83832698f1988eb27c1 (diff) | |
| download | gitlab-ce-edaa33dee2ff2f7ea3fac488d41558eb5f86d68c.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42
Diffstat (limited to 'qa')
112 files changed, 2628 insertions, 1155 deletions
diff --git a/qa/.confiner/quarantine.yml b/qa/.confiner/quarantine.yml new file mode 100644 index 00000000000..6534d72525d --- /dev/null +++ b/qa/.confiner/quarantine.yml @@ -0,0 +1,15 @@ +- name: Quarantine E2E tests that fail consistently + plugin: + name: gitlab # https://gitlab.com/gitlab-org/quality/confiner/-/blob/main/doc/plugins/gitlab.md + args: + threshold: 3 # 3 failures + private_token: $QA_GITLAB_CI_TOKEN + project_id: gitlab-org/gitlab-qa-mirror # https://gitlab.com/gitlab-org/gitlab-qa-mirror/ + target_project: gitlab-org/gitlab + failure_issue_labels: QA,Quality + failure_issue_prefix: "Failure in " + pwd: qa # E2E specs reside in the qa subdirectory + timeout: 30 + ref: master + actions: + - quarantine diff --git a/qa/Dockerfile b/qa/Dockerfile index 13213c7c8c8..54de6509518 100644 --- a/qa/Dockerfile +++ b/qa/Dockerfile @@ -77,7 +77,7 @@ COPY ./config/initializers/0_inject_enterprise_edition_module.rb /home/gitlab/co # The [b] part makes ./ee/app/models/license.r[b] a pattern that is allowed to return no files (which is the case in FOSS) COPY VERSION ./ee/app/models/license.r[b] /home/gitlab/ee/app/models/ COPY ./config/bundler_setup.rb /home/gitlab/config/ -COPY ./lib/gitlab.rb /home/gitlab/lib/ +COPY ./lib/gitlab_edition.rb /home/gitlab/lib/ COPY ./lib/gitlab/utils.rb /home/gitlab/lib/gitlab/ COPY ./INSTALLATION_TYPE ./VERSION /home/gitlab/ diff --git a/qa/Gemfile b/qa/Gemfile index 576b58f9844..c07f9dc96a7 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -25,10 +25,12 @@ gem 'octokit', '~> 4.21' gem 'webdrivers', '~> 5.0' gem 'zeitwerk', '~> 2.4' gem 'influxdb-client', '~> 1.17' -gem 'terminal-table', '~> 1.8', require: false +gem 'terminal-table', '~> 3.0.0', require: false gem 'slack-notifier', '~> 2.4', require: false gem 'fog-google', '~> 1.17', require: false +gem 'confiner', '~> 0.2' + gem 'chemlab', '~> 0.9' gem 'chemlab-library-www-gitlab-com', '~> 0.1' diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 14f10d2b047..3e85a33f2a2 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -57,6 +57,9 @@ GEM adamantium (~> 0.2.0) equalizer (~> 0.0.9) concurrent-ruby (1.1.9) + confiner (0.2.1) + gitlab (>= 4.17) + zeitwerk (~> 2.5.1) declarative (0.0.20) deprecation_toolkit (1.5.1) activesupport (>= 4.2) @@ -112,12 +115,12 @@ GEM fog-core nokogiri (>= 1.5.11, < 2.0.0) formatador (0.3.0) - gitlab (4.16.1) - httparty (~> 0.14, >= 0.14.0) - terminal-table (~> 1.5, >= 1.5.1) - gitlab-qa (7.14.0) + gitlab (4.18.0) + httparty (~> 0.18) + terminal-table (>= 1.5.1) + gitlab-qa (7.17.1) activesupport (~> 6.1) - gitlab (~> 4.16.1) + gitlab (~> 4.18.0) http (~> 5.0) nokogiri (~> 1.10) table_print (= 1.5.7) @@ -184,12 +187,12 @@ GEM memoizable (0.4.2) thread_safe (~> 0.3, >= 0.3.1) method_source (0.9.0) - mime-types (3.4.0) + mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.1115) + mime-types-data (3.2022.0105) mini_mime (1.1.0) mini_portile2 (2.6.1) - minitest (5.14.4) + minitest (5.15.0) multi_json (1.15.0) multi_xml (0.6.0) multipart-post (2.1.1) @@ -280,8 +283,8 @@ GEM slack-notifier (2.4.0) systemu (2.6.5) table_print (1.5.7) - terminal-table (1.8.0) - unicode-display_width (~> 1.1, >= 1.1.1) + terminal-table (3.0.2) + unicode-display_width (>= 1.1.1, < 3) thread_safe (0.3.6) timecop (0.9.1) trailblazer-option (0.1.2) @@ -291,7 +294,7 @@ GEM unf (0.1.4) unf_ext unf_ext (0.0.8) - unicode-display_width (1.8.0) + unicode-display_width (2.1.0) unparser (0.4.7) abstract_type (~> 0.0.7) adamantium (~> 0.2.0) @@ -312,7 +315,7 @@ GEM webrick (1.7.0) xpath (3.2.0) nokogiri (~> 1.8) - zeitwerk (2.5.1) + zeitwerk (2.5.3) PLATFORMS ruby @@ -325,6 +328,7 @@ DEPENDENCIES capybara-screenshot (~> 1.0.23) chemlab (~> 0.9) chemlab-library-www-gitlab-com (~> 0.1) + confiner (~> 0.2) deprecation_toolkit (~> 1.5.1) faker (~> 2.19, >= 2.19.0) fog-google (~> 1.17) @@ -345,10 +349,10 @@ DEPENDENCIES ruby-debug-ide (~> 0.7.0) selenium-webdriver (~> 4.0) slack-notifier (~> 2.4) - terminal-table (~> 1.8) + terminal-table (~> 3.0.0) timecop (~> 0.9.1) webdrivers (~> 5.0) zeitwerk (~> 2.4) BUNDLED WITH - 2.2.30 + 2.2.33 diff --git a/qa/Rakefile b/qa/Rakefile index e865b972b4e..5d8c49a399b 100644 --- a/qa/Rakefile +++ b/qa/Rakefile @@ -1,14 +1,9 @@ # frozen_string_literal: true # rubocop:disable Rails/RakeEnvironment -Dir['tasks/*.rake'].each { |file| load file } +require_relative "qa" -require_relative 'qa/tools/revoke_all_personal_access_tokens' -require_relative 'qa/tools/delete_subgroups' -require_relative 'qa/tools/generate_perf_testdata' -require_relative 'qa/tools/delete_test_ssh_keys' -require_relative 'qa/tools/initialize_gitlab_auth' -require_relative 'qa/tools/delete_projects' +Dir['tasks/*.rake'].each { |file| load file } desc "Revokes all personal access tokens" task :revoke_personal_access_tokens do @@ -64,4 +59,9 @@ desc "Deletes projects directly under the provided group" task :delete_projects do QA::Tools::DeleteProjects.new.run end + +desc "Deletes resources created during E2E test runs" +task :delete_test_resources, :file_pattern do |t, args| + QA::Tools::DeleteTestResources.new(args[:file_pattern]).run +end # rubocop:enable Rails/RakeEnvironment diff --git a/qa/knapsack/.gitignore b/qa/knapsack/.gitignore new file mode 100644 index 00000000000..d91d3e8efdd --- /dev/null +++ b/qa/knapsack/.gitignore @@ -0,0 +1,4 @@ +** + +!.gitignore +!master_report.json diff --git a/qa/knapsack/gcs/.gitignore b/qa/knapsack/gcs/.gitignore deleted file mode 100644 index e7c1de7e0f2..00000000000 --- a/qa/knapsack/gcs/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -** - -!.gitignore diff --git a/qa/knapsack/master_report.json b/qa/knapsack/master_report.json index 152b17a0577..0e9f67fa967 100644 --- a/qa/knapsack/master_report.json +++ b/qa/knapsack/master_report.json @@ -1,209 +1,199 @@ { - "qa/specs/features/ee/api/2_plan/epics_milestone_dates_spec.rb": 4.835599899291992, - "qa/specs/features/ee/browser_ui/3_create/repository/code_owners_spec.rb": 26.39240026473999, - "qa/specs/features/ee/browser_ui/secure/create_project_with_secure_spec.rb": 46.76790499687195, - "qa/specs/features/api/1_manage/users_spec.rb": 0.6089541912078857, - "qa/specs/features/browser_ui/1_manage/login/login_via_oauth_spec.rb": 20.58603596687317, - "qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb": 31.49540114402771, - "qa/specs/features/browser_ui/3_create/repository/add_ssh_key_spec.rb": 16.18057894706726, - "qa/specs/features/browser_ui/3_create/repository/create_edit_delete_file_via_web_spec.rb": 18.047621726989746, - "qa/specs/features/browser_ui/3_create/repository/push_over_http_file_size_spec.rb": 19.459370374679565, - "qa/specs/features/browser_ui/3_create/repository/use_ssh_key_spec.rb": 22.800872802734375, - "qa/specs/features/browser_ui/3_create/snippet/create_snippet_spec.rb": 5.812374591827393, - "qa/specs/features/browser_ui/3_create/wiki/create_edit_clone_push_wiki_spec.rb": 17.273863554000854, - "qa/specs/features/browser_ui/4_verify/ci_variable/add_ci_variable_spec.rb": 8.31815505027771, - "qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb": 51.81568956375122, - "qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb": 30.373554468154907, - "qa/specs/features/sanity/version_spec.rb": 0.0004665851593017578, - "qa/specs/features/browser_ui/3_create/repository/push_http_private_token_spec.rb": 22.993898630142212, - "qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb": 36.358747243881226, - "qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb": 9.079580068588257, - "qa/specs/features/browser_ui/3_create/repository/push_over_http_spec.rb": 39.412545919418335, - "qa/specs/features/browser_ui/3_create/repository/push_over_ssh_spec.rb": 27.905837297439575, - "qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb": 99.71209812164307, - "qa/specs/features/ee/browser_ui/2_plan/epic/promote_issue_to_epic_spec.rb": 25.72262978553772, - "qa/specs/features/ee/browser_ui/2_plan/issue_boards/project_issue_boards_spec.rb": 44.536749839782715, - "qa/specs/features/browser_ui/2_plan/milestone/create_group_milestone_spec.rb": 18.482256174087524, - "qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb": 16.314701795578003, - "qa/specs/features/ee/browser_ui/2_plan/issue_boards/read_only_board_configuration_spec.rb": 36.934654235839844, - "qa/specs/features/ee/browser_ui/2_plan/iterations/create_group_iteration_spec.rb": 17.103047847747803, - "qa/specs/features/browser_ui/2_plan/milestone/create_project_milestone_spec.rb": 28.352823495864868, - "qa/specs/features/ee/browser_ui/2_plan/issue_boards/configure_issue_board_by_label_spec.rb": 20.208287954330444, - "qa/specs/features/ee/browser_ui/2_plan/burndown_chart/burndown_chart_spec.rb": 13.733941793441772, - "qa/specs/features/browser_ui/2_plan/issue/collapse_comments_in_discussions_spec.rb": 16.376311540603638, - "qa/specs/features/ee/browser_ui/2_plan/epic/epics_management_spec.rb": 98.71593880653381, - "qa/specs/features/browser_ui/2_plan/related_issues/related_issues_spec.rb": 21.307689666748047, - "qa/specs/features/ee/browser_ui/2_plan/multiple_assignees_for_issues/more_than_four_assignees_spec.rb": 39.39231467247009, - "qa/specs/features/ee/browser_ui/2_plan/issue_boards/create_group_issue_board_spec.rb": 18.50350284576416, - "qa/specs/features/ee/browser_ui/2_plan/issues_analytics/issues_analytics_spec.rb": 30.67392325401306, - "qa/specs/features/ee/browser_ui/2_plan/custom_email/custom_email_spec.rb": 16.17888569831848, - "qa/specs/features/ee/browser_ui/2_plan/scoped_labels/editing_scoped_labels_spec.rb": 45.47245216369629, - "qa/specs/features/browser_ui/2_plan/issue/check_mentions_for_xss_spec.rb": 38.93913507461548, - "qa/specs/features/ee/browser_ui/2_plan/epic/roadmap_spec.rb": 12.087258100509644, - "qa/specs/features/ee/browser_ui/2_plan/issue_boards/group_issue_boards_spec.rb": 17.309232473373413, - "qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_dependent_relationship_spec.rb": 70.75156497955322, - "qa/specs/features/browser_ui/2_plan/issue/export_as_csv_spec.rb": 22.20275855064392, - "qa/specs/features/browser_ui/2_plan/issue/issue_suggestions_spec.rb": 24.21539831161499, - "qa/specs/features/ee/browser_ui/2_plan/issues_weight/issue_weight_visualization_spec.rb": 20.22646951675415, - "qa/specs/features/ee/browser_ui/2_plan/iterations/assign_group_iteration_spec.rb": 31.00749373435974, - "qa/specs/features/ee/browser_ui/2_plan/issue/default_issue_template_spec.rb": 18.430193424224854, - "qa/specs/features/browser_ui/3_create/wiki/project_based_page_deletion_spec.rb": 13.615312337875366, - "qa/specs/features/browser_ui/1_manage/project/protected_tags_spec.rb": 94.04865074157715, - "qa/specs/features/browser_ui/3_create/web_ide/add_new_directory_in_web_ide_spec.rb": 59.575963258743286, - "qa/specs/features/ee/browser_ui/1_manage/group/share_group_with_group_spec.rb": 25.6483793258667, - "qa/specs/features/browser_ui/3_create/merge_request/cherry_pick/cherry_pick_commit_spec.rb": 26.07162046432495, - "qa/specs/features/browser_ui/3_create/repository/protocol_v2_push_ssh_spec.rb": 8.41316819190979, - "qa/specs/features/ee/browser_ui/secure/merge_request_license_widget_spec.rb": 73.56247329711914, - "qa/specs/features/api/3_create/repository/files_spec.rb": 6.235261917114258, - "qa/specs/features/browser_ui/5_package/container_registry_spec.rb": 7.263134717941284, - "qa/specs/features/api/3_create/repository/default_branch_name_setting_spec.rb": 10.079012870788574, - "qa/specs/features/browser_ui/3_create/repository/file/delete_file_via_web_spec.rb": 16.52791404724121, - "qa/specs/features/api/3_create/repository/project_archive_compare_spec.rb": 10.684799909591675, - "qa/specs/features/browser_ui/3_create/web_ide/upload_new_file_in_web_ide_spec.rb": 72.14465713500977, - "qa/specs/features/browser_ui/3_create/snippet/create_personal_snippet_with_multiple_files_spec.rb": 15.664107322692871, - "qa/specs/features/browser_ui/3_create/snippet/add_file_to_snippet_spec.rb": 34.32576060295105, - "qa/specs/features/browser_ui/3_create/design_management/archive_design_content_spec.rb": 31.586787939071655, - "qa/specs/features/ee/browser_ui/secure/project_security_dashboard_spec.rb": 27.6742160320282, - "qa/specs/features/api/1_manage/rate_limits_spec.rb": 11.836639165878296, - "qa/specs/features/ee/browser_ui/1_manage/user/minimal_access_user_spec.rb": 15.691015005111694, - "qa/specs/features/api/3_create/merge_request/push_options_mwps_spec.rb": 38.80854368209839, - "qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb": 15.07186245918274, - "qa/specs/features/ee/browser_ui/3_create/repository/pull_mirroring_over_http_spec.rb": 41.58929800987244, - "qa/specs/features/api/3_create/repository/push_postreceive_idempotent_spec.rb": 13.438591957092285, - "qa/specs/features/ee/browser_ui/3_create/merge_request/default_merge_request_template_spec.rb": 30.66688632965088, - "qa/specs/features/browser_ui/1_manage/user/user_access_termination_spec.rb": 25.137597799301147, - "qa/specs/features/browser_ui/3_create/repository/clone_spec.rb": 8.731743574142456, - "qa/specs/features/browser_ui/7_configure/auto_devops/auto_devops_templates_spec.rb": 0.0008337497711181641, - "qa/specs/features/browser_ui/3_create/merge_request/suggestions/custom_commit_suggestion_spec.rb": 40.60329842567444, - "qa/specs/features/browser_ui/3_create/snippet/clone_push_pull_personal_snippet_spec.rb": 39.090237617492676, - "qa/specs/features/browser_ui/3_create/web_ide/create_first_file_in_web_ide_spec.rb": 22.633376359939575, - "qa/specs/features/browser_ui/3_create/design_management/modify_design_content_spec.rb": 18.457061767578125, - "qa/specs/features/browser_ui/3_create/wiki/project_based_content_creation_spec.rb": 49.273433685302734, - "qa/specs/features/browser_ui/1_manage/group/transfer_group_spec.rb": 14.6923348903656, - "qa/specs/features/browser_ui/3_create/merge_request/squash_merge_request_spec.rb": 38.44519090652466, - "qa/specs/features/browser_ui/3_create/repository/push_mirroring_lfs_over_http_spec.rb": 46.04780888557434, - "qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_via_template_spec.rb": 27.77385425567627, - "qa/specs/features/ee/browser_ui/1_manage/insights/default_insights_spec.rb": 28.850987195968628, - "qa/specs/features/browser_ui/3_create/repository/file/edit_file_via_web_spec.rb": 19.598198413848877, - "qa/specs/features/browser_ui/3_create/repository/file/file_with_unusual_name_spec.rb": 15.17819595336914, - "qa/specs/features/api/1_manage/user_access_termination_spec.rb": 6.3396782875061035, - "qa/specs/features/ee/browser_ui/secure/create_merge_request_with_secure_spec.rb": 71.55253982543945, - "qa/specs/features/browser_ui/4_verify/testing/view_code_coverage_spec.rb": 42.83795666694641, - "qa/specs/features/api/3_create/merge_request/push_options_remove_source_branch_spec.rb": 17.03721809387207, - "qa/specs/features/browser_ui/3_create/wiki/project_based_content_manipulation_spec.rb": 25.14606213569641, - "qa/specs/features/browser_ui/3_create/web_ide/open_fork_in_web_ide_spec.rb": 70.44876503944397, - "qa/specs/features/browser_ui/3_create/merge_request/merge_when_pipeline_succeeds_spec.rb": 52.93111038208008, - "qa/specs/features/browser_ui/3_create/snippet/clone_push_pull_project_snippet_spec.rb": 48.3312201499939, - "qa/specs/features/ee/browser_ui/1_manage/group/group_audit_logs_2_spec.rb": 100.1787781715393, - "qa/specs/features/browser_ui/3_create/repository/user_views_commit_diff_patch_spec.rb": 22.37237572669983, - "qa/specs/features/ee/browser_ui/1_manage/group/group_file_template_spec.rb": 69.36870694160461, - "qa/specs/features/api/3_create/merge_request/push_options_target_branch_spec.rb": 13.524356126785278, - "qa/specs/features/ee/browser_ui/1_manage/group/prevent_forking_outside_group_spec.rb": 37.17233395576477, - "qa/specs/features/api/2_plan/closes_issue_via_pushing_a_commit_spec.rb": 6.892294406890869, - "qa/specs/features/browser_ui/2_plan/issue_boards/focus_mode_spec.rb": 12.628767013549805, - "qa/specs/features/ee/browser_ui/secure/enable_sast_from_configuration_spec.rb": 86.38991785049438, - "qa/specs/features/ee/browser_ui/4_verify/pipeline_for_project_mirror_github_spec.rb": 12.684113502502441, - "qa/specs/features/browser_ui/3_create/merge_request/revert/revert_commit_spec.rb": 35.340261459350586, - "qa/specs/features/browser_ui/2_plan/issue/custom_issue_template_spec.rb": 19.719850540161133, - "qa/specs/features/browser_ui/3_create/merge_request/view_merge_request_diff_patch_spec.rb": 38.779794216156006, - "qa/specs/features/api/4_verify/cancel_pipeline_when_block_user_spec.rb": 6.86094856262207, - "qa/specs/features/browser_ui/3_create/snippet/delete_file_from_snippet_spec.rb": 51.54451298713684, - "qa/specs/features/browser_ui/3_create/repository/branch_with_unusual_name_spec.rb": 18.586050033569336, - "qa/specs/features/browser_ui/3_create/web_ide/open_web_ide_from_diff_tab_spec.rb": 35.587294578552246, - "qa/specs/features/browser_ui/1_manage/project/view_project_activity_spec.rb": 14.01547122001648, - "qa/specs/features/browser_ui/4_verify/runner/register_runner_spec.rb": 20.370421409606934, - "qa/specs/features/browser_ui/3_create/wiki/project_based_directory_management_spec.rb": 15.297787427902222, - "qa/specs/features/browser_ui/3_create/snippet/add_comment_to_snippet_spec.rb": 42.507469177246094, - "qa/specs/features/ee/browser_ui/4_verify/pipeline_status_on_operation_dashboard_spec.rb": 46.77937316894531, - "qa/specs/features/browser_ui/4_verify/pipeline/run_pipeline_via_web_only_spec.rb": 20.96509575843811, - "qa/specs/features/browser_ui/3_create/snippet/create_personal_snippet_spec.rb": 12.62861156463623, - "qa/specs/features/browser_ui/3_create/merge_request/revert/reverting_merge_request_spec.rb": 49.85329723358154, - "qa/specs/features/browser_ui/1_manage/login/log_in_with_2fa_spec.rb": 84.63548684120178, - "qa/specs/features/browser_ui/3_create/repository/protocol_v2_push_http_spec.rb": 15.584527254104614, - "qa/specs/features/browser_ui/3_create/web_ide/review_merge_request_spec.rb": 30.72457480430603, - "qa/specs/features/api/3_create/merge_request/push_options_title_description_spec.rb": 10.70707631111145, - "qa/specs/features/ee/api/1_manage/user/minimal_access_user_spec.rb": 4.82462477684021, - "qa/specs/features/browser_ui/3_create/snippet/share_snippet_spec.rb": 32.49313712120056, - "qa/specs/features/browser_ui/3_create/wiki/project_based_list_spec.rb": 32.37481236457825, - "qa/specs/features/browser_ui/2_plan/issue/filter_issue_comments_spec.rb": 16.573434114456177, - "qa/specs/features/browser_ui/3_create/repository/push_mirroring_over_http_spec.rb": 43.15674066543579, - "qa/specs/features/ee/browser_ui/secure/license_compliance_spec.rb": 31.000027179718018, - "qa/specs/features/ee/browser_ui/1_manage/project/project_audit_logs_spec.rb": 149.64519357681274, - "qa/specs/features/ee/browser_ui/1_manage/group/restrict_by_ip_address_spec.rb": 116.07316851615906, - "qa/specs/features/browser_ui/1_manage/login/register_spec.rb": 145.5431580543518, - "qa/specs/features/browser_ui/3_create/repository/file/create_file_via_web_spec.rb": 19.418848514556885, - "qa/specs/features/browser_ui/6_release/deploy_key/add_deploy_key_spec.rb": 18.54582905769348, - "qa/specs/features/api/5_package/container_registry_spec.rb": 3.93778920173645, - "qa/specs/features/ee/browser_ui/3_create/repository/restrict_push_protected_branch_spec.rb": 148.70570707321167, - "qa/specs/features/ee/browser_ui/6_release/multi-project_pipelines_spec.rb": 52.770936250686646, - "qa/specs/features/ee/browser_ui/3_create/contribution_analytics_spec.rb": 45.81806302070618, - "qa/specs/features/browser_ui/4_verify/pipeline/include_local_config_file_paths_with_wildcard_spec.rb": 20.386794805526733, - "qa/specs/features/browser_ui/3_create/repository/ssh_key_support_spec.rb": 21.968912363052368, - "qa/specs/features/browser_ui/3_create/web_ide/add_file_template_spec.rb": 71.7951602935791, - "qa/specs/features/api/1_manage/bulk_import_group_spec.rb": 50.21021842956543, - "qa/specs/features/ee/browser_ui/2_plan/issue_boards/configurable_issue_board_spec.rb": 16.42144775390625, - "qa/specs/features/ee/browser_ui/3_create/repository/merge_with_code_owner_in_root_group_spec.rb": 173.84056043624878, - "qa/specs/features/ee/browser_ui/1_manage/project/project_templates_spec.rb": 86.48439168930054, - "qa/specs/features/ee/browser_ui/3_create/repository/pull_mirroring_over_ssh_with_key_spec.rb": 56.275856018066406, - "qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_independent_relationship_spec.rb": 87.7243127822876, - "qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb": 98.70601177215576, - "qa/specs/features/ee/browser_ui/secure/vulnerability_management_spec.rb": 91.42165040969849, - "qa/specs/features/ee/browser_ui/3_create/repository/merge_with_code_owner_in_subgroup_spec.rb": 176.7654173374176, - "qa/specs/features/browser_ui/2_plan/issue/mentions_spec.rb": 17.273313283920288, - "qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb": 15.879833459854126, - "qa/specs/features/ee/browser_ui/2_plan/issue_boards/sum_of_issues_weights_spec.rb": 16.24162793159485, - "qa/specs/features/browser_ui/1_manage/group/bulk_import_group_spec.rb": 55.61779570579529, - "qa/specs/features/ee/browser_ui/2_plan/multiple_assignees_for_issues/four_assignees_spec.rb": 15.482072830200195, - "qa/specs/features/browser_ui/4_verify/pipeline/pass_dotenv_variables_to_downstream_via_bridge_spec.rb": 54.22519111633301, - "qa/specs/features/ee/browser_ui/3_create/repository/push_rules_spec.rb": 85.68487024307251, - "qa/specs/features/ee/browser_ui/3_create/web_ide/web_terminal_spec.rb": 63.027522802352905, - "qa/specs/features/ee/browser_ui/3_create/wiki/create_group_wiki_page_spec.rb": 26.17377734184265, - "qa/specs/features/browser_ui/3_create/design_management/add_design_content_spec.rb": 16.853343725204468, - "qa/specs/features/api/1_manage/project_access_token_spec.rb": 2.8215739727020264, - "qa/specs/features/ee/browser_ui/1_manage/instance/instance_audit_logs_spec.rb": 109.0079870223999, - "qa/specs/features/browser_ui/6_release/pages/pages_pipeline_spec.rb": 73.78986692428589, - "qa/specs/features/browser_ui/4_verify/pipeline/locked_artifacts_spec.rb": 34.84124970436096, - "qa/specs/features/api/1_manage/import_github_repo_spec.rb": 11.330891609191895, - "qa/specs/features/browser_ui/3_create/snippet/create_project_snippet_with_multiple_files_spec.rb": 36.70389533042908, - "qa/specs/features/browser_ui/3_create/repository/push_protected_branch_spec.rb": 25.407151699066162, - "qa/specs/features/browser_ui/7_configure/kubernetes/kubernetes_integration_spec.rb": 109.88336181640625, - "qa/specs/features/browser_ui/1_manage/user/follow_user_activity_spec.rb": 24.014917373657227, - "qa/specs/features/ee/browser_ui/3_create/repository/code_owners_with_protected_branch_and_squashed_commits_spec.rb": 42.596492767333984, - "qa/specs/features/browser_ui/3_create/snippet/snippet_index_page_spec.rb": 60.067442893981934, - "qa/specs/features/ee/browser_ui/3_create/repository/file_locking_spec.rb": 185.41551113128662, - "qa/specs/features/browser_ui/3_create/snippet/create_project_snippet_spec.rb": 30.03315234184265, - "qa/specs/features/browser_ui/2_plan/milestone/assign_milestone_spec.rb": 76.58771228790283, - "qa/specs/features/browser_ui/3_create/snippet/copy_snippet_file_contents_spec.rb": 28.64104723930359, - "qa/specs/features/browser_ui/4_verify/ci_variable/pipeline_with_protected_variable_spec.rb": 198.37600350379944, - "qa/specs/features/browser_ui/4_verify/ci_variable/add_remove_ci_variable_spec.rb": 35.098846435546875, - "qa/specs/features/browser_ui/4_verify/pipeline/trigger_matrix_spec.rb": 51.25122785568237, - "qa/specs/features/browser_ui/1_manage/group/transfer_project_spec.rb": 61.072656869888306, - "qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb": 135.90494751930237, - "qa/specs/features/ee/browser_ui/4_verify/cancelling_merge_request_in_merge_train_spec.rb": 59.57308530807495, - "qa/specs/features/browser_ui/3_create/web_ide/link_to_line_in_web_ide_spec.rb": 32.462252140045166, - "qa/specs/features/browser_ui/1_manage/project/import_github_repo_spec.rb": 53.246745586395264, - "qa/specs/features/browser_ui/4_verify/pipeline/mr_event_rule_pipeline_spec.rb": 79.99787259101868, - "qa/specs/features/browser_ui/1_manage/login/2fa_ssh_recovery_spec.rb": 57.00465273857117, - "qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb": 77.2134940624237, - "qa/specs/features/browser_ui/1_manage/login/2fa_recovery_spec.rb": 48.472514390945435, - "qa/specs/features/api/3_create/merge_request/push_options_labels_spec.rb": 17.153425455093384, - "qa/specs/features/ee/browser_ui/1_manage/group/group_audit_logs_1_spec.rb": 96.45902276039124, - "qa/specs/features/browser_ui/4_verify/pipeline/merge_mr_when_pipline_is_blocked_spec.rb": 64.81094217300415, - "qa/specs/features/browser_ui/non_devops/performance_bar_spec.rb": 27.902702569961548, - "qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_branch_switcher_spec.rb": 23.476325750350952, - "qa/specs/features/ee/browser_ui/10_protect/policy_list_spec.rb": 14.327334642410278, - "qa/specs/features/ee/browser_ui/secure/security_reports_spec.rb": 26.599382877349854, - "qa/specs/features/browser_ui/4_verify/pipeline/trigger_child_pipeline_with_manual_spec.rb": 39.33750772476196, - "qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb": 50.194467306137085, - "qa/specs/features/browser_ui/3_create/merge_request/cherry_pick/cherry_pick_a_merge_spec.rb": 59.395915508270264, - "qa/specs/features/ee/browser_ui/4_verify/pipelines_for_merged_results_and_merge_trains_spec.rb": 203.04975271224976, - "qa/specs/features/ee/browser_ui/3_create/repository/assign_code_owners_spec.rb": 42.03165292739868, - "qa/specs/features/browser_ui/1_manage/project/project_access_token_spec.rb": 18.495836973190308, - "qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb": 17.1261248588562, - "qa/specs/features/ee/browser_ui/4_verify/pipeline_subscription_with_group_owned_project_spec.rb": 53.17536449432373, - "qa/specs/features/ee/browser_ui/3_create/merge_request/add_batch_comments_in_merge_request_spec.rb": 62.42040038108826, - "qa/specs/features/browser_ui/3_create/repository/add_file_template_spec.rb": 90.16095304489136, - "qa/specs/features/ee/browser_ui/4_verify/new_discussion_not_dropping_merge_trains_mr_spec.rb": 91.91785287857056, - "qa/specs/features/browser_ui/5_package/maven_repository_spec.rb": 687.8482820987701, - "qa/specs/features/browser_ui/5_package/npm_registry_spec.rb": 225.33412504196167 + "qa/specs/features/api/1_manage/users_spec.rb": 0.38025754499994946, + "qa/specs/features/ee/browser_ui/11_fulfillment/license/license_spec.rb": 6.1726223529999515, + "qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb": 6.837727864000044, + "qa/specs/features/api/3_create/repository/files_spec.rb": 6.9398801430002095, + "qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb": 7.072401565999826, + "qa/specs/features/browser_ui/3_create/repository/clone_spec.rb": 8.430185218000133, + "qa/specs/features/ee/browser_ui/2_plan/epic/roadmap_spec.rb": 9.21629216500014, + "qa/specs/features/browser_ui/14_non_devops/service_ping_default_enabled_spec.rb": 9.460245247999865, + "qa/specs/features/api/3_create/repository/default_branch_name_setting_spec.rb": 9.749626129999797, + "qa/specs/features/browser_ui/2_plan/issue_boards/focus_mode_spec.rb": 9.997606156000074, + "qa/specs/features/api/3_create/repository/push_postreceive_idempotent_spec.rb": 10.100938496999788, + "qa/specs/features/api/3_create/merge_request/push_options_target_branch_spec.rb": 10.435720339999989, + "qa/specs/features/api/3_create/merge_request/push_options_remove_source_branch_spec.rb": 10.664103456999783, + "qa/specs/features/browser_ui/3_create/wiki/project_based_page_deletion_spec.rb": 11.010621653000044, + "qa/specs/features/browser_ui/3_create/repository/push_http_private_token_spec.rb": 11.080987325000024, + "qa/specs/features/api/3_create/merge_request/push_options_title_description_spec.rb": 11.289408692999814, + "qa/specs/features/browser_ui/3_create/snippet/create_personal_snippet_spec.rb": 11.388780440000119, + "qa/specs/features/ee/browser_ui/2_plan/burndown_chart/burndown_chart_spec.rb": 11.812601945000097, + "qa/specs/features/browser_ui/2_plan/issue/export_as_csv_spec.rb": 11.884903447000397, + "qa/specs/features/ee/browser_ui/2_plan/custom_email/custom_email_spec.rb": 11.973681912000075, + "qa/specs/features/browser_ui/3_create/wiki/project_based_directory_management_spec.rb": 11.982320482999967, + "qa/specs/features/browser_ui/3_create/repository/push_over_http_spec.rb": 12.454461346999778, + "qa/specs/features/api/2_plan/closes_issue_via_pushing_a_commit_spec.rb": 12.595205497999814, + "qa/specs/features/api/3_create/merge_request/push_options_labels_spec.rb": 12.696943737999845, + "qa/specs/features/browser_ui/3_create/repository/protocol_v2_push_http_spec.rb": 12.821059261999835, + "qa/specs/features/browser_ui/2_plan/milestone/create_group_milestone_spec.rb": 13.493086933000086, + "qa/specs/features/browser_ui/2_plan/issue/issue_suggestions_spec.rb": 13.522706569999855, + "qa/specs/features/browser_ui/3_create/repository/push_over_http_file_size_spec.rb": 13.611114558000281, + "qa/specs/features/browser_ui/1_manage/project/project_access_token_spec.rb": 13.662936730000183, + "qa/specs/features/ee/browser_ui/2_plan/issue_boards/sum_of_issues_weights_spec.rb": 13.997128957999848, + "qa/specs/features/ee/browser_ui/10_protect/policy_alerts_list_spec.rb": 14.055217947000074, + "qa/specs/features/browser_ui/3_create/snippet/create_personal_snippet_with_multiple_files_spec.rb": 14.212461153999811, + "qa/specs/features/ee/api/2_plan/epics_milestone_dates_spec.rb": 14.218627059000028, + "qa/specs/features/browser_ui/1_manage/group/transfer_group_spec.rb": 14.295524570999987, + "qa/specs/features/api/1_manage/project_access_token_spec.rb": 14.394589879999785, + "qa/specs/features/ee/browser_ui/2_plan/multiple_assignees_for_issues/four_assignees_spec.rb": 14.505683429000328, + "qa/specs/features/browser_ui/2_plan/related_issues/related_issues_spec.rb": 14.804579386000114, + "qa/specs/features/browser_ui/3_create/design_management/add_design_content_spec.rb": 14.869172961000004, + "qa/specs/features/ee/browser_ui/2_plan/iterations/assign_group_iteration_spec.rb": 15.069100258000162, + "qa/specs/features/ee/browser_ui/2_plan/issue_boards/configurable_issue_board_spec.rb": 15.071375434999936, + "qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb": 15.238976046000062, + "qa/specs/features/ee/browser_ui/2_plan/scoped_labels/editing_scoped_labels_spec.rb": 15.25893454900006, + "qa/specs/features/browser_ui/2_plan/issue/collapse_comments_in_discussions_spec.rb": 15.62486792799973, + "qa/specs/features/browser_ui/2_plan/issue/filter_issue_comments_spec.rb": 15.73652188899996, + "qa/specs/features/ee/browser_ui/2_plan/issues_analytics/issues_analytics_spec.rb": 16.175388279000117, + "qa/specs/features/ee/browser_ui/2_plan/issues_weight/issue_weight_visualization_spec.rb": 16.21438098999988, + "qa/specs/features/ee/browser_ui/2_plan/issue_boards/configure_issue_board_by_label_spec.rb": 16.22156817799987, + "qa/specs/features/ee/browser_ui/2_plan/issue_boards/group_issue_boards_spec.rb": 16.28064358199981, + "qa/specs/features/browser_ui/3_create/repository/file/delete_file_via_web_spec.rb": 16.526415993999763, + "qa/specs/features/browser_ui/3_create/wiki/content_editor_spec.rb": 16.635663933999695, + "qa/specs/features/api/1_manage/import_github_repo_spec.rb": 16.75859540500005, + "qa/specs/features/browser_ui/6_release/deploy_key/add_deploy_key_spec.rb": 16.85444325000026, + "qa/specs/features/ee/browser_ui/2_plan/epic/promote_issue_to_epic_spec.rb": 17.065811306999876, + "qa/specs/features/ee/browser_ui/11_fulfillment/purchase/user_registration_billing_spec.rb": 17.17362991999994, + "qa/specs/features/browser_ui/1_manage/project/view_project_activity_spec.rb": 17.35990919599999, + "qa/specs/features/browser_ui/4_verify/pipeline/include_local_config_file_paths_with_wildcard_spec.rb": 17.4036377489997, + "qa/specs/features/ee/browser_ui/2_plan/issue_boards/create_group_issue_board_spec.rb": 17.53151273000003, + "qa/specs/features/browser_ui/3_create/repository/ssh_key_support_spec.rb": 18.336858511999708, + "qa/specs/features/browser_ui/3_create/design_management/modify_design_content_spec.rb": 18.827946458000042, + "qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb": 19.011096268000074, + "qa/specs/features/ee/browser_ui/2_plan/issue/default_issue_template_spec.rb": 19.273840846999974, + "qa/specs/features/api/4_verify/cancel_pipeline_when_block_user_spec.rb": 19.779615910000075, + "qa/specs/features/browser_ui/4_verify/runner/register_runner_spec.rb": 19.787533178000103, + "qa/specs/features/browser_ui/2_plan/issue/custom_issue_template_spec.rb": 20.357167043000118, + "qa/specs/features/browser_ui/3_create/repository/push_protected_branch_spec.rb": 20.58807039300018, + "qa/specs/features/browser_ui/3_create/repository/protocol_v2_push_ssh_spec.rb": 20.604954283000097, + "qa/specs/features/browser_ui/3_create/repository/branch_with_unusual_name_spec.rb": 20.84536794700034, + "qa/specs/features/ee/browser_ui/3_create/wiki/create_group_wiki_page_spec.rb": 21.285323852000147, + "qa/specs/features/browser_ui/4_verify/pipeline/run_pipeline_via_web_only_spec.rb": 21.324911325999892, + "qa/specs/features/api/1_manage/user_access_termination_spec.rb": 21.359333020000122, + "qa/specs/features/browser_ui/3_create/web_ide/create_first_file_in_web_ide_spec.rb": 21.53934234799999, + "qa/specs/features/browser_ui/3_create/repository/file/file_with_unusual_name_spec.rb": 21.623363159999826, + "qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_branch_switcher_spec.rb": 21.639708403999975, + "qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_via_template_spec.rb": 22.508069357000295, + "qa/specs/features/browser_ui/3_create/wiki/project_based_content_manipulation_spec.rb": 22.67048247999992, + "qa/specs/features/ee/browser_ui/1_manage/insights/default_insights_spec.rb": 22.707359102999817, + "qa/specs/features/browser_ui/3_create/repository/file/create_file_via_web_spec.rb": 23.914098683999782, + "qa/specs/features/ee/browser_ui/1_manage/group/share_group_with_group_spec.rb": 24.228926759999922, + "qa/specs/features/browser_ui/2_plan/milestone/create_project_milestone_spec.rb": 24.320835930999692, + "qa/specs/features/browser_ui/4_verify/pipeline/mr_event_rule_pipeline_spec.rb": 24.421617836999985, + "qa/specs/features/ee/api/1_manage/user/minimal_access_user_spec.rb": 24.568071621999934, + "qa/specs/features/ee/browser_ui/2_plan/iterations/create_group_iteration_spec.rb": 25.105601008999656, + "qa/specs/features/browser_ui/3_create/repository/push_over_ssh_spec.rb": 25.670671182000206, + "qa/specs/features/ee/browser_ui/3_create/merge_request/default_merge_request_template_spec.rb": 26.588750723999965, + "qa/specs/features/browser_ui/3_create/snippet/copy_snippet_file_contents_spec.rb": 26.957921672999873, + "qa/specs/features/browser_ui/2_plan/issue/check_mentions_for_xss_spec.rb": 27.038084878000063, + "qa/specs/features/ee/browser_ui/1_manage/user/minimal_access_user_spec.rb": 27.164655879999827, + "qa/specs/features/browser_ui/3_create/repository/user_views_commit_diff_patch_spec.rb": 27.958475461000035, + "qa/specs/features/browser_ui/3_create/snippet/create_project_snippet_spec.rb": 29.15091494699982, + "qa/specs/features/api/3_create/repository/project_archive_compare_spec.rb": 29.63945763499987, + "qa/specs/features/browser_ui/3_create/wiki/project_based_list_spec.rb": 30.055674564000128, + "qa/specs/features/browser_ui/3_create/snippet/share_snippet_spec.rb": 30.36701765900034, + "qa/specs/features/browser_ui/3_create/repository/file/edit_file_via_web_spec.rb": 31.11626907400023, + "qa/specs/features/browser_ui/4_verify/pipeline/locked_artifacts_spec.rb": 31.399775493000107, + "qa/specs/features/ee/browser_ui/10_protect/policies_list_spec.rb": 31.579298365999875, + "qa/specs/features/ee/browser_ui/2_plan/multiple_assignees_for_issues/more_than_four_assignees_spec.rb": 31.699020851000114, + "qa/specs/features/ee/browser_ui/2_plan/issue_boards/read_only_board_configuration_spec.rb": 32.22923225600016, + "qa/specs/features/browser_ui/4_verify/testing/view_code_coverage_spec.rb": 32.390305334999994, + "qa/specs/features/browser_ui/3_create/snippet/add_file_to_snippet_spec.rb": 32.4001524549999, + "qa/specs/features/browser_ui/3_create/web_ide/review_merge_request_spec.rb": 32.588556773000164, + "qa/specs/features/browser_ui/4_verify/ci_variable/add_remove_ci_variable_spec.rb": 32.655282309000086, + "qa/specs/features/browser_ui/1_manage/user/follow_user_activity_spec.rb": 32.67606646000013, + "qa/specs/features/ee/browser_ui/3_create/repository/code_owners_with_protected_branch_and_squashed_commits_spec.rb": 32.86781491000011, + "qa/specs/features/browser_ui/3_create/design_management/archive_design_content_spec.rb": 33.592668581, + "qa/specs/features/browser_ui/14_non_devops/performance_bar_spec.rb": 33.844586084000184, + "qa/specs/features/browser_ui/1_manage/project/import_github_repo_spec.rb": 34.22076284800005, + "qa/specs/features/api/3_create/merge_request/push_options_mwps_spec.rb": 34.69707643500033, + "qa/specs/features/browser_ui/3_create/snippet/add_comment_to_snippet_spec.rb": 35.48510294100015, + "qa/specs/features/browser_ui/3_create/merge_request/revert/revert_commit_spec.rb": 35.60547228999985, + "qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb": 35.720513429999755, + "qa/specs/features/ee/browser_ui/3_create/repository/assign_code_owners_spec.rb": 36.74466744499978, + "qa/specs/features/browser_ui/3_create/merge_request/squash_merge_request_spec.rb": 36.86498900599986, + "qa/specs/features/browser_ui/4_verify/pipeline/trigger_matrix_spec.rb": 36.88383336300012, + "qa/specs/features/browser_ui/3_create/web_ide/open_web_ide_from_diff_tab_spec.rb": 36.91027954099991, + "qa/specs/features/browser_ui/3_create/merge_request/view_merge_request_diff_patch_spec.rb": 38.008783447999576, + "qa/specs/features/browser_ui/4_verify/pipeline/trigger_child_pipeline_with_manual_spec.rb": 38.2163332099999, + "qa/specs/features/browser_ui/1_manage/user/user_access_termination_spec.rb": 38.41997747000005, + "qa/specs/features/browser_ui/3_create/snippet/clone_push_pull_personal_snippet_spec.rb": 38.798842664000176, + "qa/specs/features/browser_ui/3_create/snippet/create_project_snippet_with_multiple_files_spec.rb": 38.86858395499985, + "qa/specs/features/ee/browser_ui/3_create/repository/code_owners_spec.rb": 39.11634360200014, + "qa/specs/features/ee/browser_ui/1_manage/group/prevent_forking_outside_group_spec.rb": 39.33498874499992, + "qa/specs/features/browser_ui/3_create/repository/push_mirroring_over_http_spec.rb": 39.408525157999975, + "qa/specs/features/browser_ui/3_create/web_ide/link_to_line_in_web_ide_spec.rb": 39.66159539199998, + "qa/specs/features/ee/browser_ui/4_verify/pipeline_status_on_operation_dashboard_spec.rb": 39.74350435799988, + "qa/specs/features/ee/browser_ui/4_verify/pipeline_subscription_with_group_owned_project_spec.rb": 39.873689517, + "qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb": 41.68991280600039, + "qa/specs/features/ee/browser_ui/3_create/repository/pull_mirroring_over_http_spec.rb": 42.34842452200019, + "qa/specs/features/ee/browser_ui/2_plan/issue_boards/project_issue_boards_spec.rb": 42.87835724299998, + "qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb": 43.00558933000002, + "qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb": 45.002151569000034, + "qa/specs/features/browser_ui/3_create/wiki/project_based_content_creation_spec.rb": 45.135388796999905, + "qa/specs/features/ee/browser_ui/3_create/merge_request/add_batch_comments_in_merge_request_spec.rb": 45.27653511099993, + "qa/specs/features/browser_ui/3_create/snippet/snippet_index_page_spec.rb": 45.49861126799988, + "qa/specs/features/browser_ui/3_create/merge_request/suggestions/custom_commit_suggestion_spec.rb": 45.95718983799998, + "qa/specs/features/browser_ui/4_verify/pipeline/pass_dotenv_variables_to_downstream_via_bridge_spec.rb": 46.274925919, + "qa/specs/features/browser_ui/3_create/snippet/clone_push_pull_project_snippet_spec.rb": 47.46850344200038, + "qa/specs/features/browser_ui/1_manage/login/2fa_ssh_recovery_spec.rb": 47.98207172000002, + "qa/specs/features/browser_ui/4_verify/pipeline/merge_mr_when_pipline_is_blocked_spec.rb": 48.11739431199976, + "qa/specs/features/browser_ui/3_create/repository/push_mirroring_lfs_over_http_spec.rb": 48.333632795000085, + "qa/specs/features/ee/browser_ui/6_release/multi-project_pipelines_spec.rb": 49.07156694399987, + "qa/specs/features/browser_ui/3_create/snippet/delete_file_from_snippet_spec.rb": 49.98554557300031, + "qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb": 50.13713323600041, + "qa/specs/features/ee/api/1_manage/bulk_import_group_spec.rb": 50.35492376299999, + "qa/specs/features/browser_ui/3_create/merge_request/cherry_pick/cherry_pick_commit_spec.rb": 50.79862357000002, + "qa/specs/features/browser_ui/3_create/merge_request/merge_when_pipeline_succeeds_spec.rb": 51.18897452500005, + "qa/specs/features/ee/browser_ui/3_create/contribution_analytics_spec.rb": 51.5182317230001, + "qa/specs/features/browser_ui/1_manage/login/2fa_recovery_spec.rb": 52.09437633100015, + "qa/specs/features/ee/browser_ui/13_secure/license_compliance_spec.rb": 53.78650449299994, + "qa/specs/features/browser_ui/3_create/merge_request/suggestions/batch_suggestion_spec.rb": 53.991341565000084, + "qa/specs/features/ee/browser_ui/13_secure/create_merge_request_with_secure_spec.rb": 56.61833271799969, + "qa/specs/features/ee/browser_ui/3_create/repository/pull_mirroring_over_ssh_with_key_spec.rb": 58.94148023099979, + "qa/specs/features/browser_ui/3_create/merge_request/cherry_pick/cherry_pick_a_merge_spec.rb": 59.88873274600019, + "qa/specs/features/browser_ui/3_create/web_ide/add_new_directory_in_web_ide_spec.rb": 61.892666940000254, + "qa/specs/features/browser_ui/3_create/web_ide/add_file_template_spec.rb": 64.76709299799995, + "qa/specs/features/browser_ui/3_create/web_ide/upload_new_file_in_web_ide_spec.rb": 65.38459789100034, + "qa/specs/features/ee/browser_ui/13_secure/merge_request_license_widget_spec.rb": 65.64657268999963, + "qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb": 66.96891640700005, + "qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_dependent_relationship_spec.rb": 68.11150705099999, + "qa/specs/features/browser_ui/1_manage/group/transfer_project_spec.rb": 70.67798023099999, + "qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb": 71.9568110780001, + "qa/specs/features/browser_ui/6_release/pages/pages_pipeline_spec.rb": 73.23625617499965, + "qa/specs/features/ee/browser_ui/1_manage/group/group_file_template_spec.rb": 74.34492043599994, + "qa/specs/features/browser_ui/1_manage/group/bulk_import_group_spec.rb": 75.80247018900013, + "qa/specs/features/browser_ui/2_plan/milestone/assign_milestone_spec.rb": 75.8654343979997, + "qa/specs/features/browser_ui/3_create/repository/add_file_template_spec.rb": 76.87336582600028, + "qa/specs/features/browser_ui/1_manage/login/log_in_with_2fa_spec.rb": 82.25273063899976, + "qa/specs/features/ee/browser_ui/2_plan/epic/epics_management_spec.rb": 82.48331730200016, + "qa/specs/features/browser_ui/1_manage/project/protected_tags_spec.rb": 84.2234556809999, + "qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_independent_relationship_spec.rb": 86.64432922500009, + "qa/specs/features/ee/browser_ui/1_manage/project/project_templates_spec.rb": 89.27986745599992, + "qa/specs/features/ee/browser_ui/4_verify/new_discussion_not_dropping_merge_trains_mr_spec.rb": 90.51838889500004, + "qa/specs/features/ee/browser_ui/4_verify/cancelling_merge_request_in_merge_train_spec.rb": 99.40036980900004, + "qa/specs/features/ee/browser_ui/1_manage/group/group_audit_logs_1_spec.rb": 99.88996194999982, + "qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb": 100.75571312800002, + "qa/specs/features/browser_ui/2_plan/issue/mentions_spec.rb": 102.73961456400002, + "qa/specs/features/ee/browser_ui/4_verify/pipelines_for_merged_results_and_merge_trains_spec.rb": 108.17952484600005, + "qa/specs/features/browser_ui/3_create/web_ide/open_fork_in_web_ide_spec.rb": 108.78527251000014, + "qa/specs/features/ee/browser_ui/1_manage/group/restrict_by_ip_address_spec.rb": 108.98986279500014, + "qa/specs/features/ee/browser_ui/3_create/repository/merge_with_code_owner_in_root_group_spec.rb": 109.23582328299972, + "qa/specs/features/browser_ui/4_verify/pipeline/update_ci_file_with_pipeline_editor_spec.rb": 117.43055826199998, + "qa/specs/features/ee/browser_ui/3_create/repository/push_rules_spec.rb": 120.7805862900002, + "qa/specs/features/ee/browser_ui/13_secure/security_reports_spec.rb": 121.40739863099998, + "qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb": 121.93682936799996, + "qa/specs/features/ee/browser_ui/1_manage/group/group_audit_logs_2_spec.rb": 124.35619795699995, + "qa/specs/features/ee/browser_ui/1_manage/instance/instance_audit_logs_spec.rb": 129.314630158, + "qa/specs/features/ee/browser_ui/13_secure/vulnerability_management_spec.rb": 132.3623656059999, + "qa/specs/features/ee/browser_ui/13_secure/enable_scanning_from_configuration_spec.rb": 135.19990866599983, + "qa/specs/features/browser_ui/4_verify/ci_variable/pipeline_with_protected_variable_spec.rb": 136.47575045699978, + "qa/specs/features/ee/browser_ui/13_secure/project_security_dashboard_spec.rb": 137.37273152800026, + "qa/specs/features/ee/browser_ui/1_manage/project/project_audit_logs_spec.rb": 154.7787124900001, + "qa/specs/features/ee/browser_ui/3_create/repository/restrict_push_protected_branch_spec.rb": 161.17543870600002, + "qa/specs/features/api/1_manage/bulk_import_group_spec.rb": 167.9717091839998, + "qa/specs/features/ee/browser_ui/3_create/repository/merge_with_code_owner_in_subgroup_spec.rb": 192.326786567, + "qa/specs/features/browser_ui/1_manage/login/register_spec.rb": 412.09824056499974, + "qa/specs/features/ee/browser_ui/3_create/repository/file_locking_spec.rb": 428.72626845000013, + "qa/specs/features/api/1_manage/bulk_import_project_spec.rb": 686.55653471, + "qa/specs/features/api/1_manage/rate_limits_spec.rb": 923.7095398920001 } diff --git a/qa/lib/gitlab/page/main/welcome.rb b/qa/lib/gitlab/page/main/welcome.rb deleted file mode 100644 index a2df1da61c9..00000000000 --- a/qa/lib/gitlab/page/main/welcome.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Page - module Main - class Welcome < Chemlab::Page - path '/users/sign_up/welcome' - - button :get_started_button - end - end - end -end diff --git a/qa/lib/gitlab/page/main/welcome.stub.rb b/qa/lib/gitlab/page/main/welcome.stub.rb deleted file mode 100644 index a10e697bcbf..00000000000 --- a/qa/lib/gitlab/page/main/welcome.stub.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Page - module Main - module Welcome - # @note Defined as +button :get_started_button+ - # Clicks +get_started_button+ - def get_started_button - # This is a stub, used for indexing. The method is dynamically generated. - end - - # @example - # Gitlab::Page::Main::Welcome.perform do |welcome| - # expect(welcome.get_started_button_element).to exist - # end - # @return [Watir::Button] The raw +Button+ element - def get_started_button_element - # This is a stub, used for indexing. The method is dynamically generated. - end - - # @example - # Gitlab::Page::Main::Welcome.perform do |welcome| - # expect(welcome).to be_get_started_button - # end - # @return [Boolean] true if the +get_started_button+ element is present on the page - def get_started_button? - # This is a stub, used for indexing. The method is dynamically generated. - end - end - end - end -end @@ -2,7 +2,7 @@ Encoding.default_external = 'UTF-8' -require_relative '../lib/gitlab' +require_relative '../lib/gitlab_edition' require_relative '../lib/gitlab/utils' require_relative '../config/initializers/0_inject_enterprise_edition_module' diff --git a/qa/qa/fixtures/metrics_dashboards/templating.yml b/qa/qa/fixtures/metrics_dashboards/templating.yml index e06e7cc1247..847eba59bd2 100644 --- a/qa/qa/fixtures/metrics_dashboards/templating.yml +++ b/qa/qa/fixtures/metrics_dashboards/templating.yml @@ -40,4 +40,4 @@ panel_groups: - id: pod_memory_working_set1 query_range: 'container_memory_working_set_bytes{pod_name=~"{{pod_name2}}"}/1024/1024' unit: "MiB" - label: pod_name + label: pod_name
\ No newline at end of file diff --git a/qa/qa/flow/login.rb b/qa/qa/flow/login.rb index 5f7e0227ac5..b60f74fe9bf 100644 --- a/qa/qa/flow/login.rb +++ b/qa/qa/flow/login.rb @@ -6,8 +6,6 @@ module QA module_function def while_signed_in(as: nil, address: :gitlab, admin: false) - Page::Main::Menu.perform(&:sign_out_if_signed_in) - sign_in(as: as, address: address, admin: admin) result = yield @@ -23,9 +21,10 @@ module QA end def sign_in(as: nil, address: :gitlab, skip_page_validation: false, admin: false) + Page::Main::Login.perform { |p| p.redirect_to_login_page(address) } + unless Page::Main::Login.perform(&:on_login_page?) Page::Main::Menu.perform(&:sign_out) if Page::Main::Menu.perform(&:signed_in?) - Runtime::Browser.visit(address, Page::Main::Login) end Page::Main::Login.perform do |login| diff --git a/qa/qa/flow/sign_up.rb b/qa/qa/flow/sign_up.rb index a2a62371092..ec7886ef969 100644 --- a/qa/qa/flow/sign_up.rb +++ b/qa/qa/flow/sign_up.rb @@ -26,9 +26,12 @@ module QA sign_up.click_new_user_register_button end - Page::Registration::Welcome.perform(&:click_get_started_button_if_available) + Flow::UserOnboarding.onboard_user success = if user.expect_fabrication_success + # In development env and .com the user is asked to create a group and a project which can be skipped for + # the purpose of signing up + Runtime::Browser.visit(:gitlab, Page::Dashboard::Welcome) Page::Main::Menu.perform(&:has_personal_area?) else Page::Main::Menu.perform(&:not_signed_in?) diff --git a/qa/qa/flow/user_onboarding.rb b/qa/qa/flow/user_onboarding.rb new file mode 100644 index 00000000000..066e1878869 --- /dev/null +++ b/qa/qa/flow/user_onboarding.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module QA + module Flow + module UserOnboarding + module_function + + def onboard_user + Page::Registration::Welcome.perform do |welcome_page| + if welcome_page.has_get_started_button? + welcome_page.select_role('Other') + welcome_page.choose_setup_for_company_if_available + welcome_page.click_get_started_button + end + end + end + end + end +end diff --git a/qa/qa/page/base.rb b/qa/qa/page/base.rb index 69f58dcb8a5..526dd25ccc9 100644 --- a/qa/qa/page/base.rb +++ b/qa/qa/page/base.rb @@ -386,6 +386,10 @@ module QA end end + def current_host + URI(page.current_url).host + end + def self.path raise NotImplementedError end diff --git a/qa/qa/page/component/confirm_modal.rb b/qa/qa/page/component/confirm_modal.rb index 25eea8e0d93..a90be76c879 100644 --- a/qa/qa/page/component/confirm_modal.rb +++ b/qa/qa/page/component/confirm_modal.rb @@ -8,12 +8,6 @@ module QA def self.included(base) super - - base.view 'app/views/shared/_confirm_modal.html.haml' do - element :confirm_modal - element :confirm_input - element :confirm_button - end end def fill_confirmation_text(text) diff --git a/qa/qa/page/component/invite_members_modal.rb b/qa/qa/page/component/invite_members_modal.rb index fecd61fb410..ca6862ccb02 100644 --- a/qa/qa/page/component/invite_members_modal.rb +++ b/qa/qa/page/component/invite_members_modal.rb @@ -47,37 +47,43 @@ module QA fill_element :members_token_select_input, username Support::WaitForRequests.wait_for_requests click_button username - - # Guest option is selected by default, skipping these steps if desired option is 'Guest' - unless access_level == 'Guest' - click_element :access_level_dropdown - click_button access_level - end - - click_element :invite_button + set_access_level(access_level) end - Support::WaitForRequests.wait_for_requests - - page.refresh + send_invite end - def invite_group(group_name, group_access = Resource::Members::AccessLevel::GUEST) + def invite_group(group_name, access_level = 'Guest') open_invite_group_modal - fill_element :access_level_dropdown, with: group_access + within_element(:invite_members_modal_content) do + click_button 'Select a group' - click_button 'Select a group' - fill_element :group_select_dropdown_search_field, group_name + # Helps stabilize race condition with concurrent group API calls while searching + # TODO: Replace with `fill_element :group_select_dropdown_search_field, group_name` when this bug is resolved: https://gitlab.com/gitlab-org/gitlab/-/issues/349379 + send_keys_to_element(:group_select_dropdown_search_field, group_name) - Support::WaitForRequests.wait_for_requests + Support::WaitForRequests.wait_for_requests + click_button group_name + set_access_level(access_level) + end - click_button group_name + send_invite + end - click_element :invite_button + private - Support::WaitForRequests.wait_for_requests + def set_access_level(access_level) + # Guest option is selected by default, skipping these steps if desired option is 'Guest' + unless access_level == 'Guest' + click_element :access_level_dropdown + click_button access_level + end + end + def send_invite + click_element :invite_button + Support::WaitForRequests.wait_for_requests page.refresh end end diff --git a/qa/qa/page/dashboard/welcome.rb b/qa/qa/page/dashboard/welcome.rb index b54205780d9..6f645e168c7 100644 --- a/qa/qa/page/dashboard/welcome.rb +++ b/qa/qa/page/dashboard/welcome.rb @@ -11,6 +11,10 @@ module QA def has_welcome_title?(text) has_element?(:welcome_title_content, text: text) end + + def self.path + '/' + end end end end diff --git a/qa/qa/page/file/show.rb b/qa/qa/page/file/show.rb index 730c5a88515..e54c3e0cd07 100644 --- a/qa/qa/page/file/show.rb +++ b/qa/qa/page/file/show.rb @@ -23,8 +23,26 @@ module QA element :delete_file_button, "button_tag 'Delete file'" # rubocop:disable QA/ElementWithPattern end + view 'app/assets/javascripts/vue_shared/components/web_ide_link.vue' do + element :edit_button + end + + view 'app/assets/javascripts/vue_shared/components/actions_button.vue' do + element :action_dropdown + element :edit_menu_item, ':data-qa-selector="`${action.key}_menu_item`"' # rubocop:disable QA/ElementWithPattern + end + def click_edit - click_on 'Edit' + # TODO: remove this condition and else part once ff :consolidated_edit_button is enabled by default + if has_element?(:action_dropdown) + within_element(:action_dropdown) do + click_button(class: 'dropdown-toggle-split') + click_element(:edit_menu_item) + click_element(:edit_button) + end + else + click_on 'Edit' + end end def click_delete diff --git a/qa/qa/page/main/login.rb b/qa/qa/page/main/login.rb index 5cba9d4bce4..f004107d7bd 100644 --- a/qa/qa/page/main/login.rb +++ b/qa/qa/page/main/login.rb @@ -156,6 +156,11 @@ module QA sign_in_using_credentials(user: user) end + def redirect_to_login_page(address) + desired_host = URI(Runtime::Scenario.send("#{address}_address")).host + Runtime::Browser.visit(address, Page::Main::Login) if desired_host != current_host + end + private def sign_in_using_gitlab_credentials(user:, skip_page_validation: false) diff --git a/qa/qa/page/project/branches/show.rb b/qa/qa/page/project/branches/show.rb index afec0e27a0b..a19fcf8ec6e 100644 --- a/qa/qa/page/project/branches/show.rb +++ b/qa/qa/page/project/branches/show.rb @@ -14,7 +14,6 @@ module QA end view 'app/views/projects/branches/_branch.html.haml' do - element :remove_btn element :branch_name end diff --git a/qa/qa/page/project/members.rb b/qa/qa/page/project/members.rb index eeb589d6ca8..1102abd6646 100644 --- a/qa/qa/page/project/members.rb +++ b/qa/qa/page/project/members.rb @@ -41,6 +41,11 @@ module QA click_button 'Remove group' end end + + def has_group?(group_name) + click_element :groups_list_tab + has_element?(:group_row, text: group_name) + end end end end diff --git a/qa/qa/page/project/new.rb b/qa/qa/page/project/new.rb index 5ff52527774..42baf1f3f87 100644 --- a/qa/qa/page/project/new.rb +++ b/qa/qa/page/project/new.rb @@ -14,7 +14,6 @@ module QA view 'app/views/projects/_new_project_fields.html.haml' do element :initialize_with_readme_checkbox element :initialize_with_sast_checkbox - element :project_namespace_field, 'namespaces_options' # rubocop:disable QA/ElementWithPattern element :project_name, 'text_field :name' # rubocop:disable QA/ElementWithPattern element :project_path, 'text_field :path' # rubocop:disable QA/ElementWithPattern element :project_description, 'text_area :description' # rubocop:disable QA/ElementWithPattern diff --git a/qa/qa/page/project/packages/show.rb b/qa/qa/page/project/packages/show.rb index 4872c0bc705..5ba9ad7df40 100644 --- a/qa/qa/page/project/packages/show.rb +++ b/qa/qa/page/project/packages/show.rb @@ -5,7 +5,7 @@ module QA module Project module Packages class Show < QA::Page::Base - view 'app/assets/javascripts/packages_and_registries/package_registry/components/details/app.vue' do + view 'app/assets/javascripts/packages_and_registries/package_registry/pages/details.vue' do element :delete_button element :delete_modal_button element :package_information_content diff --git a/qa/qa/page/project/pipeline_editor/show.rb b/qa/qa/page/project/pipeline_editor/show.rb index e430884ea08..8289039d4c5 100644 --- a/qa/qa/page/project/pipeline_editor/show.rb +++ b/qa/qa/page/project/pipeline_editor/show.rb @@ -24,6 +24,27 @@ module QA element :source_editor_container, require: true end + view 'app/assets/javascripts/pipeline_editor/components/header/pipeline_status.vue' do + element :pipeline_id_content + end + + view 'app/assets/javascripts/pipeline_editor/components/commit/commit_form.vue' do + element :commit_changes_button + end + + view 'app/assets/javascripts/pipeline_editor/components/header/validation_segment.vue' do + element :validation_message_content + end + + view 'app/assets/javascripts/pipelines/components/pipeline_graph/pipeline_graph.vue' do + element :stage_container + element :job_container + end + + view 'app/assets/javascripts/pipeline_editor/components/pipeline_editor_tabs.vue' do + element :file_editor_container + end + def initialize super @@ -50,8 +71,70 @@ module QA find_element(:source_editor_container).text end + def write_to_editor(text) + find_element(:source_editor_container).fill_in(with: text) + end + + def submit_changes + click_element(:commit_changes_button) + + wait_for_requests + end + + def set_target_branch(name) + find_element(:target_branch_field).fill_in(with: name) + end + + def current_branch + find_element(:branch_selector_button).text + end + + def pipeline_id + find_element(:pipeline_id_content).text.delete!('#').to_i + end + + def ci_syntax_validate_message + find_element(:validation_message_content).text + end + + def go_to_visualize_tab + go_to_tab('Visualize') + end + + def go_to_lint_tab + go_to_tab('Lint') + end + + def go_to_view_merged_yaml_tab + go_to_tab('View merged YAML') + end + + def has_source_editor? + has_element?(:source_editor_container) + end + + def has_stage?(name) + all_elements(:stage_container, minimum: 1).any? { |item| item.text.match(/#{name}/i) } + end + + def has_job?(name) + all_elements(:job_container, minimum: 1).any? { |item| item.text.match(/#{name}/i) } + end + + def tab_alert_message + within_element(:file_editor_container) do + find('.gl-alert-body').text + end + end + private + def go_to_tab(name) + within_element(:file_editor_container) do + find('.nav-item', text: name).click + end + end + # If the page thinks user has never opened pipeline editor before # It will expand pipeline editor sidebar by default # Collapse the sidebar if it is expanded diff --git a/qa/qa/page/project/secure/configuration_form.rb b/qa/qa/page/project/secure/configuration_form.rb index 3e89a57e870..fa1fad44273 100644 --- a/qa/qa/page/project/secure/configuration_form.rb +++ b/qa/qa/page/project/secure/configuration_form.rb @@ -8,6 +8,10 @@ module QA include QA::Page::Component::Select2 include QA::Page::Settings::Common + view 'app/assets/javascripts/security_configuration/components/app.vue' do + element :security_configuration_history_link + end + view 'app/assets/javascripts/security_configuration/components/feature_card.vue' do element :dependency_scanning_status, "`${feature.type}_status`" # rubocop:disable QA/ElementWithPattern element :sast_status, "`${feature.type}_status`" # rubocop:disable QA/ElementWithPattern @@ -15,6 +19,22 @@ module QA element :dependency_scanning_mr_button, "`${feature.type}_mr_button`" # rubocop:disable QA/ElementWithPattern end + view 'app/assets/javascripts/security_configuration/components/auto_dev_ops_alert.vue' do + element :autodevops_container + end + + def has_security_configuration_history_link? + has_element?(:security_configuration_history_link) + end + + def has_no_security_configuration_history_link? + has_no_element?(:security_configuration_history_link) + end + + def click_security_configuration_history_link + click_element(:security_configuration_history_link) + end + def click_sast_enable_button click_element(:sast_enable_button) end @@ -29,11 +49,37 @@ module QA end end + def has_no_sast_status?(status_text) + within_element(:sast_status) do + has_no_text?(status_text) + end + end + def has_dependency_scanning_status?(status_text) within_element(:dependency_scanning_status) do has_text?(status_text) end end + + def has_no_dependency_scanning_status?(status_text) + within_element(:dependency_scanning_status) do + has_no_text?(status_text) + end + end + + def has_auto_devops_container? + has_element?(:autodevops_container) + end + + def has_no_auto_devops_container? + has_no_element?(:autodevops_container) + end + + def has_auto_devops_container_description? + within_element(:autodevops_container) do + has_text?('Quickly enable all continuous testing and compliance tools by enabling Auto DevOps') + end + end end end end diff --git a/qa/qa/page/project/settings/visibility_features_permissions.rb b/qa/qa/page/project/settings/visibility_features_permissions.rb index 1d6686ae360..60cea6de7f5 100644 --- a/qa/qa/page/project/settings/visibility_features_permissions.rb +++ b/qa/qa/page/project/settings/visibility_features_permissions.rb @@ -5,14 +5,9 @@ module QA module Project module Settings class VisibilityFeaturesPermissions < Page::Base - include QA::Page::Component::Select2 - - view 'app/views/projects/edit.html.haml' do - element :visibility_features_permissions_save_button - end - view 'app/assets/javascripts/pages/projects/shared/permissions/components/settings_panel.vue' do element :project_visibility_dropdown + element :visibility_features_permissions_save_button end def set_project_visibility(visibility) diff --git a/qa/qa/page/registration/sign_up.rb b/qa/qa/page/registration/sign_up.rb index 6d1b9cb3615..4fedc05c702 100644 --- a/qa/qa/page/registration/sign_up.rb +++ b/qa/qa/page/registration/sign_up.rb @@ -16,10 +16,6 @@ module QA element :new_user_username_field end - view 'app/views/registrations/welcome/show.html.haml' do - element :get_started_button - end - def fill_new_user_first_name_field(first_name) fill_element :new_user_first_name_field, first_name end diff --git a/qa/qa/page/registration/welcome.rb b/qa/qa/page/registration/welcome.rb index ff22e62b63e..660b33b4f5b 100644 --- a/qa/qa/page/registration/welcome.rb +++ b/qa/qa/page/registration/welcome.rb @@ -6,14 +6,25 @@ module QA class Welcome < Page::Base view 'app/views/registrations/welcome/show.html.haml' do element :get_started_button + element :role_dropdown end - def click_get_started_button_if_available - if has_element?(:get_started_button) - Support::Retrier.retry_until do - click_element :get_started_button - has_no_element?(:get_started_button) - end + def has_get_started_button? + has_element?(:get_started_button) + end + + def select_role(role) + select_element(:role_dropdown, role) + end + + def choose_setup_for_company_if_available + # Only implemented in EE + end + + def click_get_started_button + Support::Retrier.retry_until do + click_element :get_started_button + has_no_element?(:get_started_button) end end end diff --git a/qa/qa/page/trials/new.rb b/qa/qa/page/trials/new.rb index 268f3b71717..6e9d7fce688 100644 --- a/qa/qa/page/trials/new.rb +++ b/qa/qa/page/trials/new.rb @@ -6,17 +6,13 @@ module QA class New < Chemlab::Page path '/-/trials/new' - # TODO: Supplant with data-qa-selectors - text_field :first_name, id: 'first_name' - text_field :last_name, id: 'last_name' - text_field :company_name, id: 'company_name' - select :number_of_employees, id: 'company_size' - text_field :telephone_number, id: 'phone_number' - text_field :number_of_users, id: 'number_of_users' - - select :country, id: 'country_select' - - button :continue, value: 'Continue' + text_field :first_name + text_field :last_name + text_field :company_name + select :number_of_employees + text_field :telephone_number + select :country + button :continue end end end diff --git a/qa/qa/resource/api_fabricator.rb b/qa/qa/resource/api_fabricator.rb index 6315ef0bd22..4c77c515cfd 100644 --- a/qa/qa/resource/api_fabricator.rb +++ b/qa/qa/resource/api_fabricator.rb @@ -63,6 +63,10 @@ module QA process_api_response(parse_body(response)) end + def api_fabrication_http_method + @api_fabrication_http_method ||= :post + end + private def resource_web_url(resource) @@ -85,6 +89,8 @@ module QA raise ResourceNotFoundError, "Resource at #{request.mask_url} could not be found (#{response.code}): `#{response}`." end + @api_fabrication_http_method = :get # rubocop:disable Gitlab/ModuleWithInstanceVariables + response end diff --git a/qa/qa/resource/base.rb b/qa/qa/resource/base.rb index 640d2a8f06e..0112e766cf0 100644 --- a/qa/qa/resource/base.rb +++ b/qa/qa/resource/base.rb @@ -71,34 +71,49 @@ module QA resource_web_url = yield resource.web_url = resource_web_url + QA::Tools::TestResourceDataProcessor.collect(resource, resource_identifier(resource)) + resource end + def resource_identifier(resource) + if resource.respond_to?(:username) && resource.username + "with username '#{resource.username}'" + elsif resource.respond_to?(:full_path) && resource.full_path + "with full_path '#{resource.full_path}'" + elsif resource.respond_to?(:name) && resource.name + "with name '#{resource.name}'" + elsif resource.respond_to?(:id) && resource.id + "with id '#{resource.id}'" + elsif resource.respond_to?(:iid) && resource.iid + "with iid '#{resource.iid}'" + end + rescue QA::Resource::Base::NoValueError + nil + end + def log_fabrication(method, resource, parents, args) start = Time.now Support::FabricationTracker.start_fabrication result = yield.tap do fabrication_time = Time.now - start - resource_identifier = begin - if resource.respond_to?(:username) && resource.username - "with username '#{resource.username}'" - elsif resource.respond_to?(:full_path) && resource.full_path - "with full_path '#{resource.full_path}'" - elsif resource.respond_to?(:name) && resource.name - "with name '#{resource.name}'" - elsif resource.respond_to?(:id) && resource.id - "with id '#{resource.id}'" - end - rescue QA::Resource::Base::NoValueError - nil - end + + fabrication_http_method = if resource.api_fabrication_http_method == :get + if self.include?(Reusable) + "Retrieved for reuse" + else + "Retrieved" + end + else + "Built" + end Support::FabricationTracker.save_fabrication(:"#{method}_fabrication", fabrication_time) Runtime::Logger.debug do msg = ["==#{'=' * parents.size}>"] - msg << "Built a #{name}" - msg << resource_identifier if resource_identifier + msg << "#{fabrication_http_method} a #{name}" + msg << resource_identifier(resource) if resource_identifier(resource) msg << "as a dependency of #{parents.last}" if parents.any? msg << "via #{method}" msg << "in #{fabrication_time} seconds" @@ -156,11 +171,11 @@ module QA raise NotImplementedError end - def visit! + def visit!(skip_resp_code_check: false) Runtime::Logger.debug(%(Visiting #{self.class.name} at "#{web_url}")) # Just in case an async action is not yet complete - Support::WaitForRequests.wait_for_requests + Support::WaitForRequests.wait_for_requests(skip_resp_code_check: skip_resp_code_check) Support::Retrier.retry_until do visit(web_url) @@ -168,7 +183,7 @@ module QA end # Wait until the new page is ready for us to interact with it - Support::WaitForRequests.wait_for_requests + Support::WaitForRequests.wait_for_requests(skip_resp_code_check: skip_resp_code_check) end def populate(*attribute_names) @@ -179,6 +194,30 @@ module QA QA::Support::Waiter.wait_until(max_duration: max_duration, sleep_interval: sleep_interval, &block) end + # Object comparison + # + # @param [QA::Resource::Base] other + # @return [Boolean] + def ==(other) + other.is_a?(self.class) && comparable == other.comparable + end + + # Override inspect for a better rspec failure diff output + # + # @return [String] + def inspect + JSON.pretty_generate(comparable) + end + + protected + + # Custom resource comparison logic using resource attributes from api_resource + # + # @return [Hash] + def comparable + raise("comparable method needs to be implemented in order to compare resources via '=='") + end + private def attribute_value(name, block) diff --git a/qa/qa/resource/bulk_import_group.rb b/qa/qa/resource/bulk_import_group.rb index e8dc2d291b8..a22529152e1 100644 --- a/qa/qa/resource/bulk_import_group.rb +++ b/qa/qa/resource/bulk_import_group.rb @@ -3,18 +3,21 @@ module QA module Resource class BulkImportGroup < Group - attributes :source_group_path, + attributes :source_group, + :destination_group, :import_id - attribute :destination_group_path do - source_group_path - end - attribute :access_token do api_client.personal_access_token end - alias_method :path, :source_group_path + # In most cases we will want to set path the same as source group + # but it can be set to a custom name as well when imported via API + attribute :destination_group_path do + source_group.path + end + # Can't define path as attribue since @path is set in base class initializer + alias_method :path, :destination_group_path delegate :gitlab_address, to: 'QA::Runtime::Scenario' @@ -51,9 +54,9 @@ module QA entities: [ { source_type: 'group_entity', - source_full_path: source_group_path, + source_full_path: source_group.full_path, destination_name: destination_group_path, - destination_namespace: sandbox.path + destination_namespace: sandbox.full_path } ] } diff --git a/qa/qa/resource/group_badge.rb b/qa/qa/resource/group_badge.rb index fd76f066e8b..3719b502b93 100644 --- a/qa/qa/resource/group_badge.rb +++ b/qa/qa/resource/group_badge.rb @@ -39,27 +39,12 @@ module QA # @return [String] def resource_web_url(_resource); end - # Object comparison - # - # @param [QA::Resource::GroupBadge] other - # @return [Boolean] - def ==(other) - other.is_a?(GroupBadge) && comparable_badge == other.comparable_badge - end - - # Override inspect for a better rspec failure diff output - # - # @return [String] - def inspect - JSON.pretty_generate(comparable_badge) - end - protected # Return subset of fields for comparing badges # # @return [Hash] - def comparable_badge + def comparable reload! unless api_response api_response.slice( diff --git a/qa/qa/resource/group_base.rb b/qa/qa/resource/group_base.rb index 19bb5ea00d7..9f492a046db 100644 --- a/qa/qa/resource/group_base.rb +++ b/qa/qa/resource/group_base.rb @@ -123,18 +123,12 @@ module QA end # Object comparison + # Override to make sure we are comparing descendands of GroupBase # # @param [QA::Resource::GroupBase] other # @return [Boolean] def ==(other) - other.is_a?(GroupBase) && comparable_group == other.comparable_group - end - - # Override inspect for a better rspec failure diff output - # - # @return [String] - def inspect - JSON.pretty_generate(comparable_group) + other.is_a?(GroupBase) && comparable == other.comparable end protected @@ -142,7 +136,7 @@ module QA # Return subset of fields for comparing groups # # @return [Hash] - def comparable_group + def comparable reload! if api_response.nil? api_resource.slice( diff --git a/qa/qa/resource/group_milestone.rb b/qa/qa/resource/group_milestone.rb index 880ca2b9721..b9ec53e929c 100644 --- a/qa/qa/resource/group_milestone.rb +++ b/qa/qa/resource/group_milestone.rb @@ -56,27 +56,12 @@ module QA end end - # Object comparison - # - # @param [QA::Resource::GroupMilestone] other - # @return [Boolean] - def ==(other) - other.is_a?(GroupMilestone) && comparable_milestone == other.comparable_milestone - end - - # Override inspect for a better rspec failure diff output - # - # @return [String] - def inspect - JSON.pretty_generate(comparable_milestone) - end - protected # Return subset of fields for comparing milestones # # @return [Hash] - def comparable_milestone + def comparable reload! unless api_response api_response.slice( diff --git a/qa/qa/resource/issue.rb b/qa/qa/resource/issue.rb index 344f177932f..1e38de97c1e 100644 --- a/qa/qa/resource/issue.rb +++ b/qa/qa/resource/issue.rb @@ -3,7 +3,7 @@ module QA module Resource class Issue < Base - attr_writer :description, :milestone, :template, :weight + attr_writer :milestone, :template, :weight attribute :project do Project.fabricate! do |resource| @@ -95,19 +95,13 @@ module QA ) end - # Object comparison + # Create a new comment # - # @param [QA::Resource::Issue] other - # @return [Boolean] - def ==(other) - other.is_a?(Issue) && comparable_issue == other.comparable_issue - end - - # Override inspect for a better rspec failure diff output - # - # @return [String] - def inspect - JSON.pretty_generate(comparable_issue) + # @param [String] body + # @param [Boolean] confidential + # @return [Hash] + def add_comment(body:, confidential: false) + api_post_to(api_comments_path, body: body, confidential: confidential) end protected @@ -115,7 +109,7 @@ module QA # Return subset of fields for comparing issues # # @return [Hash] - def comparable_issue + def comparable reload! if api_response.nil? api_resource.slice( diff --git a/qa/qa/resource/label_base.rb b/qa/qa/resource/label_base.rb index b1af0e23561..855e41af8bb 100644 --- a/qa/qa/resource/label_base.rb +++ b/qa/qa/resource/label_base.rb @@ -49,27 +49,12 @@ module QA } end - # Object comparison - # - # @param [QA::Resource::GroupBase] other - # @return [Boolean] - def ==(other) - other.is_a?(LabelBase) && comparable_label == other.comparable_label - end - - # Override inspect for a better rspec failure diff output - # - # @return [String] - def inspect - JSON.pretty_generate(comparable_label) - end - protected # Return subset of fields for comparing labels # # @return [Hash] - def comparable_label + def comparable reload! unless api_response api_response.slice( diff --git a/qa/qa/resource/merge_request.rb b/qa/qa/resource/merge_request.rb index ba63e0823f0..685cccea02d 100644 --- a/qa/qa/resource/merge_request.rb +++ b/qa/qa/resource/merge_request.rb @@ -6,11 +6,13 @@ module QA attr_accessor :approval_rules, :source_branch, :target_new_branch, + :update_existing_file, :assignee, :milestone, :labels, :file_name, :file_content + attr_writer :no_preparation, :wait_for_merge, :template @@ -25,6 +27,8 @@ module QA attribute :project do Project.fabricate_via_api! do |resource| resource.name = 'project-with-merge-request' + resource.initialize_with_readme = true + resource.api_client = api_client end end @@ -33,22 +37,27 @@ module QA end attribute :target do - Repository::ProjectPush.fabricate! do |resource| + Repository::Commit.fabricate_via_api! do |resource| resource.project = project - resource.branch_name = target_branch - resource.new_branch = target_new_branch - resource.remote_branch = target_branch + resource.api_client = api_client + resource.commit_message = 'This is a test commit' + resource.add_files([{ 'file_path': "file-#{SecureRandom.hex(8)}.txt", 'content': 'MR init' }]) + resource.branch = target_branch + + resource.start_branch = project.default_branch if target_branch != project.default_branch end end attribute :source do - Repository::ProjectPush.fabricate! do |resource| + Repository::Commit.fabricate_via_api! do |resource| resource.project = project - resource.branch_name = target_branch - resource.remote_branch = source_branch - resource.new_branch = false - resource.file_name = file_name - resource.file_content = file_content + resource.api_client = api_client + resource.commit_message = 'This is a test commit' + resource.branch = source_branch + resource.start_branch = target_branch + + files = [{ 'file_path': file_name, 'content': file_content }] + update_existing_file ? resource.update_files(files) : resource.add_files(files) end end @@ -63,6 +72,7 @@ module QA @file_name = "added_file-#{SecureRandom.hex(8)}.txt" @file_content = "File Added" @target_new_branch = true + @update_existing_file = false @no_preparation = false @wait_for_merge = true end @@ -168,27 +178,18 @@ module QA ) end - # Object comparison - # - # @param [QA::Resource::MergeRequest] other - # @return [Boolean] - def ==(other) - other.is_a?(MergeRequest) && comparable_mr == other.comparable_mr - end - - # Override inspect for a better rspec failure diff output + # Add mr comment # - # @return [String] - def inspect - JSON.pretty_generate(comparable_mr) + # @param [String] body + # @return [Hash] + def add_comment(body) + api_post_to(api_comments_path, body: body) end - protected - # Return subset of fields for comparing merge requests # # @return [Hash] - def comparable_mr + def comparable reload! if api_response.nil? api_resource.except( @@ -197,7 +198,9 @@ module QA :project_id, :source_project_id, :target_project_id, + :merge_status, # these can differ depending on user fetching mr + :user, :subscribed, :first_contribution ).merge({ references: api_resource[:references].except(:full) }) @@ -211,8 +214,24 @@ module QA super(api_resource) end + # Create source and target and commits if necessary + # + # @return [void] def populate_target_and_source_if_required - populate(:target, :source) unless @no_preparation + return if @no_preparation + + populate(:target) if create_target? + populate(:source) + end + + # Check if target needs to be created + # + # Return false if project was already initialized and mr target is default branch + # Return false if target_new_branch is explicitly set to false + # + # @return [Boolean] + def create_target? + !(project.initialize_with_readme && target_branch == project.default_branch) && target_new_branch end end end diff --git a/qa/qa/resource/project.rb b/qa/qa/resource/project.rb index d3c1e91f358..0750ea49224 100644 --- a/qa/qa/resource/project.rb +++ b/qa/qa/resource/project.rb @@ -107,32 +107,6 @@ module QA super end - def has_file?(file_path) - response = repository_tree - - raise ResourceNotFoundError, (response[:message]).to_s if response.is_a?(Hash) && response.has_key?(:message) - - response.any? { |file| file[:path] == file_path } - end - - def has_branch?(branch) - has_branches?(Array(branch)) - end - - def has_branches?(branches) - branches.all? do |branch| - response = get(request_url("#{api_repository_branches_path}/#{branch}")) - response.code == HTTP_STATUS_OK - end - end - - def has_tags?(tags) - tags.all? do |tag| - response = get(request_url("#{api_repository_tags_path}/#{tag}")) - response.code == HTTP_STATUS_OK - end - end - def api_get_path "/projects/#{CGI.escape(path_with_namespace)}" end @@ -237,6 +211,32 @@ module QA post_body end + def has_file?(file_path) + response = repository_tree + + raise ResourceNotFoundError, (response[:message]).to_s if response.is_a?(Hash) && response.has_key?(:message) + + response.any? { |file| file[:path] == file_path } + end + + def has_branch?(branch) + has_branches?(Array(branch)) + end + + def has_branches?(branches) + branches.all? do |branch| + response = get(request_url("#{api_repository_branches_path}/#{branch}")) + response.code == HTTP_STATUS_OK + end + end + + def has_tags?(tags) + tags.all? do |tag| + response = get(request_url("#{api_repository_tags_path}/#{tag}")) + response.code == HTTP_STATUS_OK + end + end + def change_repository_storage(new_storage) response = put(request_url(api_put_path), repository_storage: new_storage) @@ -372,27 +372,12 @@ module QA api_post_to(api_wikis_path, title: title, content: content) end - # Object comparison - # - # @param [QA::Resource::Project] other - # @return [Boolean] - def ==(other) - other.is_a?(Project) && comparable_project == other.comparable_project - end - - # Override inspect for a better rspec failure diff output - # - # @return [String] - def inspect - JSON.pretty_generate(comparable_project) - end - protected # Return subset of fields for comparing projects # # @return [Hash] - def comparable_project + def comparable reload! if api_response.nil? api_resource.slice( diff --git a/qa/qa/resource/repository/commit.rb b/qa/qa/resource/repository/commit.rb index f5dba164de6..54093a5c195 100644 --- a/qa/qa/resource/repository/commit.rb +++ b/qa/qa/resource/repository/commit.rb @@ -22,42 +22,7 @@ module QA def initialize @commit_message = 'QA Test - Commit message' - end - - def add_files(files) - validate_files!(files) - - @add_files = files - end - - def add_directory(dir) - raise "Must set directory as a Pathname" unless dir.is_a?(Pathname) - - files_to_add = [] - - dir.each_child do |child| - case child.ftype? - when "file" - files_to_add.append({ - file_path: child.to_s, - content: child.read - }) - when "directory" - add_directory(child) - else - continue - end - end - - validate_files!(files_to_add) - - @add_files.merge(files_to_add) - end - - def update_files(files) - validate_files!(files) - - @update_files = files + @actions = [] end # If `actions` are specified, it performs the actions to create, @@ -72,32 +37,74 @@ module QA end def api_get_path - api_post_path + "/projects/#{CGI.escape(project.path_with_namespace)}/repository/commits" end def api_post_path - "/projects/#{CGI.escape(project.path_with_namespace)}/repository/commits" + api_get_path end def api_post_body { - branch: @branch || project.default_branch, - author_email: @author_email || Runtime::User.default_email, - author_name: @author_name || Runtime::User.username, + branch: branch || project.default_branch, + author_email: author_email || api_client.user&.email || Runtime::User.default_email, + author_name: author_name || api_client.user&.name || Runtime::User.username, commit_message: commit_message, actions: actions }.merge(new_branch) end - def actions - pending_actions = [] - pending_actions << @add_files.map { |file| file.merge({ action: "create" }) } if @add_files - pending_actions << @update_files.map { |file| file.merge({ action: "update" }) } if @update_files - pending_actions.flatten + # Add files + # Pass in array of new files like, example: + # [{ "file_path": "foo/bar", "content": "some content" }] + # + # @param [Array<Hash>] files + # @return [void] + def add_files(files) + validate_files!(files) + + actions.push(*files.map { |file| file.merge({ action: "create" }) }) + end + + # Update files + # Pass in array of files and it's contents, example: + # [{ "file_path": "foo/bar", "content": "some content" }] + # + # @param [Array<Hash>] files + # @return [void] + def update_files(files) + validate_files!(files) + + actions.push(*files.map { |file| file.merge({ action: "update" }) }) + end + + # Add all files from directory + # + # @param [Pathname] dir + # @return [void] + def add_directory(dir) + raise "Must set directory as a Pathname" unless dir.is_a?(Pathname) + + files_to_add = [] + + dir.each_child do |child| + case child.ftype + when "directory" + add_directory(child) + when "file" + files_to_add.push({ file_path: child.basename, content: child.read }) + else + continue + end + end + + add_files(files_to_add) end private + attr_reader :actions + def validate_files!(files) if !files.is_a?(Array) || files.empty? || diff --git a/qa/qa/resource/reusable_group.rb b/qa/qa/resource/reusable_group.rb new file mode 100644 index 00000000000..a4bd799e85c --- /dev/null +++ b/qa/qa/resource/reusable_group.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module QA + module Resource + class ReusableGroup < Group + prepend Reusable + + def initialize + super + + @path = "reusable_group" + @description = "QA reusable group" + @reuse_as = :default_group + end + + # Confirms that the group can be reused + # + # @return [nil] returns nil unless an error is raised + def validate_reuse_preconditions + unless reused_path_unique? + raise ResourceReuseError, + "Reusable groups must have the same name. The group reused as #{reuse_as} has the path '#{path}' but it should be '#{self.class.resources[reuse_as].path}'" + end + end + + # Confirms that reuse of the resource did not change it in a way that breaks later reuse. This raises an error if + # the current group path doesn't match the original path. + def validate_reuse + reload! + + if api_resource[:path] != @path + raise ResourceReuseError, "The group now has the path '#{api_resource[:path]}' but it should be '#{path}'" + end + end + + # Checks if the group is being reused with the same path. + # + # @return [Boolean] true if the group's path is different from another group with the same reuse symbol (reuse_as) + def reused_path_unique? + return true unless self.class.resources.key?(reuse_as) + + self.class.resources[reuse_as].path == path + end + + # Overrides QA::Resource::Group#remove_via_api! to log a debug message stating that removal will happen after + # the suite completes rather than now. + # + # @return [nil] + def remove_via_api! + QA::Runtime::Logger.debug("#{self.class.name} - deferring removal until after suite") + end + end + end +end diff --git a/qa/qa/resource/reusable_project.rb b/qa/qa/resource/reusable_project.rb index 6b33bb9d65d..d2dfff8ad56 100644 --- a/qa/qa/resource/reusable_project.rb +++ b/qa/qa/resource/reusable_project.rb @@ -5,6 +5,12 @@ module QA class ReusableProject < Project prepend Reusable + attribute :group do + ReusableGroup.fabricate_via_api! do |resource| + resource.api_client = api_client + end + end + def initialize super diff --git a/qa/qa/resource/user.rb b/qa/qa/resource/user.rb index ed4ea057484..eab428a61e7 100644 --- a/qa/qa/resource/user.rb +++ b/qa/qa/resource/user.rb @@ -98,6 +98,12 @@ module QA super end + def exists? + api_get + rescue ResourceNotFoundError + false + end + def api_delete super @@ -181,6 +187,15 @@ module QA ) end + protected + + # Compare users by username and password + # + # @return [Array] + def comparable + [username, password] + end + private def ldap_post_body diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb index b73199f1fdd..1679698a9c0 100644 --- a/qa/qa/runtime/env.rb +++ b/qa/qa/runtime/env.rb @@ -9,6 +9,7 @@ module QA extend self attr_writer :personal_access_token, :admin_personal_access_token + attr_accessor :dry_run ENV_VARIABLES = Gitlab::QA::Runtime::Env::ENV_VARIABLES @@ -89,6 +90,20 @@ module QA enabled?(ENV['ACCEPT_INSECURE_CERTS']) end + def running_on_dot_com? + uri = URI.parse(Runtime::Scenario.gitlab_address) + uri.host.include?('.com') + end + + def running_on_dev? + uri = URI.parse(Runtime::Scenario.gitlab_address) + uri.port != 80 && uri.port != 443 + end + + def running_on_dev_or_dot_com? + running_on_dev? || running_on_dot_com? + end + def running_in_ci? ENV['CI'] || ENV['CI_SERVER'] end @@ -281,9 +296,7 @@ module QA end def knapsack? - return false unless ENV['CI_NODE_TOTAL'].to_i > 1 - - !!(ENV['KNAPSACK_GENERATE_REPORT'] || ENV['KNAPSACK_REPORT_PATH'] || ENV['KNAPSACK_TEST_FILE_PATTERN']) + ENV['CI_NODE_TOTAL'].to_i > 1 && ENV['NO_KNAPSACK'] != "true" end def ldap_username @@ -401,7 +414,7 @@ module QA end def gitlab_agentk_version - ENV.fetch('GITLAB_AGENTK_VERSION', 'v14.4.0') + ENV.fetch('GITLAB_AGENTK_VERSION', 'v14.5.0') end def transient_trials @@ -416,6 +429,11 @@ module QA running_in_ci? && enabled?(ENV['QA_EXPORT_TEST_METRICS'], default: true) end + def test_resources_created_filepath + file_name = running_in_ci? ? "test-resources-#{SecureRandom.hex(3)}.json" : 'test-resources.json' + ENV.fetch('QA_TEST_RESOURCES_CREATED_FILEPATH', File.join(Path.qa_root, 'tmp', file_name)) + end + private def remote_grid_credentials diff --git a/qa/qa/scenario/bootable.rb b/qa/qa/scenario/bootable.rb index ae180ffce1c..2a9bbbc9fdb 100644 --- a/qa/qa/scenario/bootable.rb +++ b/qa/qa/scenario/bootable.rb @@ -31,6 +31,13 @@ module QA end next + elsif opt.name == :count_examples_only + parser.on(opt.arg, opt.desc) do |value| + QA::Runtime::Env.dry_run = true + Runtime::Scenario.define(opt.name, value) + end + + next end parser.on(opt.arg, opt.desc) do |value| diff --git a/qa/qa/scenario/shared_attributes.rb b/qa/qa/scenario/shared_attributes.rb index ddbe28f05d9..d5d7aedb47f 100644 --- a/qa/qa/scenario/shared_attributes.rb +++ b/qa/qa/scenario/shared_attributes.rb @@ -13,6 +13,7 @@ module QA 'Specify FEATURE_FLAGS as comma-separated flag=state pairs, e.g., "flag1=enabled,flag2=disabled"' attribute :parallel, '--parallel', 'Execute tests in parallel' attribute :loop, '--loop', 'Execute test repeatedly' + attribute :count_examples_only, '--count-examples-only', 'Return the number of examples without running them' end end end diff --git a/qa/qa/scenario/template.rb b/qa/qa/scenario/template.rb index 50bb952f1fd..ef634d3ccda 100644 --- a/qa/qa/scenario/template.rb +++ b/qa/qa/scenario/template.rb @@ -33,9 +33,15 @@ module QA Runtime::Scenario.define(:about_address, URI(-> { gitlab_address.host = "about.#{gitlab_address.host}"; gitlab_address }.call).to_s) # rubocop:disable Style/Semicolon ## + # Setup knapsack and download latest report + # + Tools::KnapsackReport.configure! if Runtime::Env.knapsack? + + ## # Perform before hooks, which are different for CE and EE # - Runtime::Release.perform_before_hooks + + Runtime::Release.perform_before_hooks unless Runtime::Env.dry_run Runtime::Feature.enable(options[:enable_feature]) if options.key?(:enable_feature) Runtime::Feature.disable(options[:disable_feature]) if options.key?(:disable_feature) && (@feature_enabled = Runtime::Feature.enabled?(options[:disable_feature])) diff --git a/qa/qa/service/praefect_manager.rb b/qa/qa/service/praefect_manager.rb index dd4cce5d0b0..7e47049d446 100644 --- a/qa/qa/service/praefect_manager.rb +++ b/qa/qa/service/praefect_manager.rb @@ -19,7 +19,7 @@ module QA @virtual_storage = 'default' end - attr_reader :primary_node, :secondary_node, :tertiary_node + attr_reader :primary_node, :secondary_node, :tertiary_node, :postgres # Executes the praefect `dataloss` command. # @@ -83,19 +83,19 @@ module QA def start_node(name) shell "docker start #{name}" + end + + def stop_node(name) + shell "docker stop #{name}" wait_until_shell_command_matches( "docker inspect -f {{.State.Running}} #{name}", - /true/, + /false/, sleep_interval: 3, max_duration: 180, retry_on_exception: true ) end - def stop_node(name) - shell "docker stop #{name}" - end - def clear_replication_queue QA::Runtime::Logger.info("Clearing the replication queue") shell sql_to_docker_exec_cmd( @@ -174,13 +174,13 @@ module QA end def start_all_nodes + start_node(@postgres) start_node(@primary_node) start_node(@secondary_node) start_node(@tertiary_node) start_node(@praefect) wait_for_health_check_all_nodes - wait_for_reliable_connection end def verify_storage_move(source_storage, destination_storage, repo_type: :project) @@ -192,21 +192,23 @@ module QA end def wait_for_praefect - wait_until_shell_command_matches( - "docker inspect -f {{.State.Running}} #{@praefect}", - /true/, - sleep_interval: 3, - max_duration: 180, - retry_on_exception: true - ) + QA::Runtime::Logger.info("Waiting for health check on praefect") + Support::Waiter.wait_until(max_duration: 120, sleep_interval: 1, raise_on_failure: true) do + # praefect runs a grpc server on port 2305, which will return an error 'Connection refused' until such time it is ready + wait_until_shell_command("docker exec #{@gitaly_cluster} bash -c 'curl #{@praefect}:2305'") do |line| + break if line.include?('curl: (1) Received HTTP/0.9 when not allowed') - QA::Runtime::Logger.info('Wait until Praefect starts and is listening') - wait_until_shell_command_matches( - "docker exec #{@praefect} bash -c 'cat /var/log/gitlab/praefect/current'", - /listening at tcp address/ - ) + QA::Runtime::Logger.debug(line.chomp) + end + end + end - wait_for_gitaly_check + def praefect_sql_ping_healthy? + cmd = "docker exec #{@praefect} bash -c '/opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml sql-ping'" + wait_until_shell_command(cmd) do |line| + QA::Runtime::Logger.debug(line.chomp) + break line.include?('praefect sql-ping: OK') + end end def wait_for_sql_ping @@ -220,32 +222,7 @@ module QA ['error when pinging healthcheck', 'failed checking node health'].include?(msg) end - def wait_for_no_praefect_storage_error - # If a healthcheck error was the last message to be logged, we'll keep seeing that message even if it's no longer a problem - # That is, there's no message shown in the Praefect logs when the healthcheck succeeds - # To work around that we perform the gitaly check rake task, wait a few seconds, and then we confirm that no healthcheck errors appear - - QA::Runtime::Logger.info("Checking that Praefect does not report healthcheck errors with its gitaly nodes") - - Support::Waiter.wait_until(max_duration: 120) do - wait_for_gitaly_check - - sleep 5 - - shell "docker exec #{@praefect} bash -c 'tail -n 1 /var/log/gitlab/praefect/current'" do |line| - QA::Runtime::Logger.debug(line.chomp) - log = JSON.parse(line) - - break true unless health_check_failure_message?(log['msg']) - rescue JSON::ParserError - # Ignore lines that can't be parsed as JSON - end - end - end - - def wait_for_storage_nodes - wait_for_no_praefect_storage_error - + def wait_for_dial_nodes_successful Support::Waiter.repeat_until(max_attempts: 3, max_duration: 120, sleep_interval: 1) do nodes_confirmed = { @primary_node => false, @@ -253,39 +230,55 @@ module QA @tertiary_node => false } - wait_until_shell_command("docker exec #{@praefect} bash -c '/opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dial-nodes'") do |line| - QA::Runtime::Logger.debug(line.chomp) + nodes_confirmed.each_key do |node| + nodes_confirmed[node] = true if praefect_dial_nodes_status?(node) + end - nodes_confirmed.each_key do |node| - nodes_confirmed[node] = true if line =~ /SUCCESS: confirmed Gitaly storage "#{node}" in virtual storages \[#{@virtual_storage}\] is served/ - end + nodes_confirmed.values.all? + end + end - nodes_confirmed.values.all? + def praefect_dial_nodes_status?(node, expect_healthy = true) + cmd = "docker exec #{@praefect} bash -c '/opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dial-nodes -timeout 1s'" + if expect_healthy + wait_until_shell_command_matches(cmd, /SUCCESS: confirmed Gitaly storage "#{node}" in virtual storages \[#{@virtual_storage}\] is served/) + else + wait_until_shell_command(cmd, raise_on_failure: false) do |line| + QA::Runtime::Logger.debug(line.chomp) + break true if line.include?('the following nodes are not healthy') && line.include?(node) end end end def wait_for_health_check_all_nodes - wait_for_health_check(@primary_node) - wait_for_health_check(@secondary_node) - wait_for_health_check(@tertiary_node) + wait_for_gitaly_health_check(@primary_node) + wait_for_gitaly_health_check(@secondary_node) + wait_for_gitaly_health_check(@tertiary_node) end - def wait_for_health_check(node) + def wait_for_gitaly_health_check(node) QA::Runtime::Logger.info("Waiting for health check on #{node}") + Support::Waiter.wait_until(max_duration: 120, sleep_interval: 1, raise_on_failure: true) do + # gitaly runs a grpc server on port 8075, which will return an error 'Connection refused' until such time it is ready + wait_until_shell_command("docker exec #{@praefect} bash -c 'curl #{node}:8075'") do |line| + break if line.include?('curl: (1) Received HTTP/0.9 when not allowed') + + QA::Runtime::Logger.debug(line.chomp) + end + end wait_until_node_is_marked_as_healthy_storage(node) end def wait_for_primary_node_health_check - wait_for_health_check(@primary_node) + wait_for_gitaly_health_check(@primary_node) end def wait_for_secondary_node_health_check - wait_for_health_check(@secondary_node) + wait_for_gitaly_health_check(@secondary_node) end def wait_for_tertiary_node_health_check - wait_for_health_check(@tertiary_node) + wait_for_gitaly_health_check(@tertiary_node) end def wait_for_health_check_failure(node) @@ -311,7 +304,6 @@ module QA shell sql_to_docker_exec_cmd("SELECT count(*) FROM healthy_storages WHERE storage = '#{node}';") do |line| result << line end - QA::Runtime::Logger.debug("result is ---#{result}") result[2].to_i == 0 end end @@ -322,21 +314,10 @@ module QA shell sql_to_docker_exec_cmd("SELECT count(*) FROM healthy_storages WHERE storage = '#{node}';") do |line| result << line end - - QA::Runtime::Logger.debug("result is ---#{result}") result[2].to_i == 1 end end - def wait_for_gitaly_check - Support::Waiter.wait_until(max_duration: 120, sleep_interval: 1, raise_on_failure: true) do - wait_until_shell_command("docker exec #{@gitlab} bash -c 'gitlab-rake gitlab:git:fsck'") do |line| - QA::Runtime::Logger.debug(line.chomp) - line.include?('Done') - end - end - end - # Waits until there is an increase in the number of reads for # any node compared to the number of reads provided. If a node # has no pre-read data, consider it to have had zero reads. @@ -354,12 +335,6 @@ module QA data.find(-> {{ value: 0 }}) { |item| item[:node] == node }[:value] end - def wait_for_reliable_connection - QA::Runtime::Logger.info('Wait until GitLab and Praefect can communicate reliably') - wait_for_sql_ping - wait_for_storage_nodes - end - def wait_for_replication(project_id) Support::Waiter.wait_until(sleep_interval: 1) { replication_queue_incomplete_count == 0 && replicated?(project_id) } end @@ -414,10 +389,34 @@ module QA end def remove_tracked_praefect_repository(relative_path, virtual_storage) - cmd = "gitlab-ctl praefect remove-repository --repository-relative-path #{relative_path} --virtual-storage-name #{virtual_storage}" + cmd = "gitlab-ctl praefect remove-repository --repository-relative-path #{relative_path} --virtual-storage-name #{virtual_storage} --apply" shell "docker exec #{@praefect} bash -c '#{cmd}'" end + # set_replication_factor assigns or unassigns random storage nodes as necessary to reach the desired replication factor for a repository + def set_replication_factor(relative_path, virtual_storage, factor) + cmd = "/opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml set-replication-factor -repository #{relative_path} -virtual-storage #{virtual_storage} -replication-factor #{factor}" + shell "docker exec #{@praefect} bash -c '#{cmd}'" + end + + # get_replication_storages retrieves a list of currently assigned storages for a repository + def get_replication_storages(relative_path, virtual_storage) + storage_repositories = [] + query = "SELECT storage FROM repository_assignments WHERE relative_path='#{relative_path}' AND virtual_storage='#{virtual_storage}';" + shell(sql_to_docker_exec_cmd(query)) { |line| storage_repositories << line.strip } + # Returned data from query will be in format + # storage + # -------- + # gitaly1 + # gitaly3 + # gitaly2 + # (3 rows) + # + + # remove 2 header rows and last 2 rows from query response (including blank line) + storage_repositories[2..-3] + end + def add_repo_to_disk(node, repo_path) cmd = "GIT_DIR=. git init --initial-branch=main /var/opt/gitlab/git-data/repositories/#{repo_path}" shell "docker exec --user git #{node} bash -c '#{cmd}'" @@ -452,7 +451,7 @@ module QA end def repository_replicated_to_disk?(node, relative_path) - Support::Waiter.wait_until(max_duration: 300, sleep_interval: 3, raise_on_failure: false) do + Support::Waiter.wait_until(max_duration: 300, sleep_interval: 1, raise_on_failure: false) do result = [] shell sql_to_docker_exec_cmd("SELECT count(*) FROM storage_repositories where relative_path='#{relative_path}';") do |line| result << line diff --git a/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_group_spec.rb index 799a5f7eaf2..a6655471591 100644 --- a/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_group_spec.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true module QA - # run only base UI validation on staging because test requires top level group creation which is problematic - # on staging environment - RSpec.describe 'Manage', :requires_admin, except: { subdomain: :staging } do + RSpec.describe 'Manage', :requires_admin do describe 'Gitlab migration' do let(:import_wait_duration) { { max_duration: 300, sleep_interval: 2 } } let(:admin_api_client) { Runtime::API::Client.as_admin } @@ -22,9 +20,18 @@ module QA end end + let(:destination_group) do + Resource::Group.fabricate_via_api! do |group| + group.api_client = api_client + group.sandbox = sandbox + group.path = "destination-group-for-import-#{SecureRandom.hex(4)}" + end + end + let(:source_group) do - Resource::Sandbox.fabricate_via_api! do |group| + Resource::Group.fabricate_via_api! do |group| group.api_client = api_client + group.sandbox = sandbox group.path = "source-group-for-import-#{SecureRandom.hex(4)}" group.avatar = File.new('qa/fixtures/designs/tanuki.jpg', 'r') end @@ -33,8 +40,8 @@ module QA let(:imported_group) do Resource::BulkImportGroup.fabricate_via_api! do |group| group.api_client = api_client - group.sandbox = sandbox - group.source_group_path = source_group.path + group.sandbox = destination_group + group.source_group = source_group end end @@ -167,7 +174,6 @@ module QA expect { imported_group.import_status }.to eventually_eq('finished').within(import_wait_duration) imported_member = imported_group.reload!.members.find { |usr| usr.username == member.username } - aggregate_failures do expect(imported_member).not_to be_nil expect(imported_member.access_level).to eq(Resource::Members::AccessLevel::DEVELOPER) diff --git a/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_issue_spec.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_issue_spec.rb new file mode 100644 index 00000000000..8a2a382ac45 --- /dev/null +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_issue_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require_relative 'gitlab_project_migration_common' + +module QA + RSpec.describe 'Manage', :requires_admin do + describe 'Gitlab migration', quarantine: { + only: { job: 'praefect' }, + type: :investigating, + issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/348999' + } do + include_context 'with gitlab project migration' + + context 'with project issues' do + let!(:source_issue) do + Resource::Issue.fabricate_via_api! do |issue| + issue.api_client = api_client + issue.project = source_project + issue.labels = %w[label_one label_two] + end + end + + let!(:source_comment) { source_issue.add_comment(body: 'This is a test comment!') } + + let(:imported_issues) { imported_projects.first.issues } + + let(:imported_issue) do + issue = imported_issues.first + Resource::Issue.init do |resource| + resource.api_client = api_client + resource.project = imported_projects.first + resource.iid = issue[:iid] + end + end + + let(:imported_comments) { imported_issue.comments } + + it( + 'successfully imports issue', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347608' + ) do + expect_import_finished + + aggregate_failures do + expect(imported_issues.count).to eq(1) + expect(imported_issue).to eq(source_issue.reload!) + + expect(imported_comments.count).to eq(1) + expect(imported_comments.first[:body]).to include(source_comment[:body]) + end + end + end + end + end +end diff --git a/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_mr_spec.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_mr_spec.rb new file mode 100644 index 00000000000..9dce9bff3c1 --- /dev/null +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_mr_spec.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require_relative 'gitlab_project_migration_common' + +module QA + RSpec.describe 'Manage', :requires_admin do + describe 'Gitlab migration', quarantine: { + only: { job: 'praefect' }, + type: :investigating, + issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/348999' + } do + include_context 'with gitlab project migration' + + context 'with merge request' do + let!(:source_project_with_readme) { true } + + let!(:other_user) do + Resource::User + .fabricate_via_api! { |usr| usr.api_client = admin_api_client } + .tap do |usr| + usr.set_public_email + source_project.add_member(usr, Resource::Members::AccessLevel::MAINTAINER) + end + end + + let!(:source_mr) do + Resource::MergeRequest.fabricate_via_api! do |mr| + mr.project = source_project + mr.api_client = Runtime::API::Client.new(user: other_user) + end + end + + let!(:source_comment) { source_mr.add_comment('This is a test comment!') } + + let(:imported_mrs) { imported_project.merge_requests } + let(:imported_mr_comments) { imported_mr.comments } + + let(:imported_mr) do + Resource::MergeRequest.init do |mr| + mr.project = imported_project + mr.iid = imported_mrs.first[:iid] + mr.api_client = api_client + end + end + + after do + other_user.remove_via_api! + end + + it( + 'successfully imports merge request', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348478' + ) do + expect_import_finished + + aggregate_failures do + expect(imported_mrs.count).to eq(1) + # TODO: remove custom comparison after member migration is implemented + # https://gitlab.com/gitlab-org/gitlab/-/issues/341886 + expect(imported_mr.comparable.except(:author)).to eq(source_mr.reload!.comparable.except(:author)) + + expect(imported_mr_comments.count).to eq(1) + expect(imported_mr_comments.first[:body]).to include(source_comment[:body]) + # Comment will have mention of original user since members are not migrated yet + expect(imported_mr_comments.first[:body]).to include(other_user.name) + end + end + end + end + end +end diff --git a/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_project_spec.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_project_spec.rb new file mode 100644 index 00000000000..a0c758c99e6 --- /dev/null +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_project_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require_relative 'gitlab_project_migration_common' + +module QA + RSpec.describe 'Manage', :requires_admin do + describe 'Gitlab migration', quarantine: { + only: { job: 'praefect' }, + type: :investigating, + issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/348999' + } do + include_context 'with gitlab project migration' + + context 'with uninitialized project' do + it( + 'successfully imports project', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347610' + ) do + expect_import_finished + + expect(imported_projects.first).to eq(source_project) + end + end + + context 'with repository' do + let(:source_project_with_readme) { true } + let(:source_commits) { source_project.commits.map { |c| c.except(:web_url) } } + let(:source_tags) do + source_project.repository_tags.tap do |tags| + tags.each { |t| t[:commit].delete(:web_url) } + end + end + + let(:source_branches) do + source_project.repository_branches.tap do |branches| + branches.each do |b| + b.delete(:web_url) + b[:commit].delete(:web_url) + end + end + end + + let(:imported_commits) { imported_project.commits.map { |c| c.except(:web_url) } } + let(:imported_tags) do + imported_project.repository_tags.tap do |tags| + tags.each { |t| t[:commit].delete(:web_url) } + end + end + + let(:imported_branches) do + imported_project.repository_branches.tap do |branches| + branches.each do |b| + b.delete(:web_url) + b[:commit].delete(:web_url) + end + end + end + + before do + source_project.create_repository_branch('test-branch') + source_project.create_repository_tag('v0.0.1') + end + + it( + 'successfully imports repository', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347570' + ) do + expect_import_finished + + aggregate_failures do + expect(imported_commits).to match_array(source_commits) + expect(imported_tags).to match_array(source_tags) + expect(imported_branches).to match_array(source_branches) + end + end + end + + context 'with wiki' do + before do + source_project.create_wiki_page(title: 'Import test project wiki', content: 'Wiki content') + end + + it( + 'successfully imports project wiki', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347567' + ) do + expect_import_finished + + expect(imported_projects.first.wikis).to eq(source_project.wikis) + end + end + end + end +end diff --git a/qa/qa/specs/features/api/1_manage/migration/gitlab_project_migration_common.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_project_migration_common.rb new file mode 100644 index 00000000000..827ebc1f5e2 --- /dev/null +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_project_migration_common.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module QA + RSpec.shared_context 'with gitlab project migration' do + let(:source_project_with_readme) { false } + let(:import_wait_duration) { { max_duration: 300, sleep_interval: 2 } } + let(:admin_api_client) { Runtime::API::Client.as_admin } + let(:user) do + Resource::User.fabricate_via_api! do |usr| + usr.api_client = admin_api_client + usr.hard_delete_on_api_removal = true + end + end + + let(:api_client) { Runtime::API::Client.new(user: user) } + + let(:sandbox) do + Resource::Sandbox.fabricate_via_api! do |group| + group.api_client = admin_api_client + end + end + + let(:destination_group) do + Resource::Group.fabricate_via_api! do |group| + group.api_client = api_client + group.sandbox = sandbox + group.path = "destination-group-for-import-#{SecureRandom.hex(4)}" + end + end + + let(:source_group) do + Resource::Group.fabricate_via_api! do |group| + group.api_client = api_client + group.path = "source-group-for-import-#{SecureRandom.hex(4)}" + end + end + + let(:source_project) do + Resource::Project.fabricate_via_api! do |project| + project.api_client = api_client + project.group = source_group + project.initialize_with_readme = source_project_with_readme + end + end + + let(:imported_group) do + Resource::BulkImportGroup.fabricate_via_api! do |group| + group.api_client = api_client + group.sandbox = destination_group + group.source_group = source_group + end + end + + let(:imported_projects) { imported_group.reload!.projects } + let(:imported_project) { imported_projects.first } + + let(:import_failures) do + imported_group.import_details.sum([]) { |details| details[:failures] } + end + + def expect_import_finished + imported_group # trigger import + + expect { imported_group.import_status }.to eventually_eq('finished').within(import_wait_duration) + expect(imported_projects.count).to eq(1), 'Expected to have 1 imported project' + end + + before do + Runtime::Feature.enable(:bulk_import_projects) + + sandbox.add_member(user, Resource::Members::AccessLevel::MAINTAINER) + source_project # fabricate source group and project + end + + after do |example| + # Checking for failures in the test currently makes test very flaky + # Just log in case of failure until cause of network errors is found + Runtime::Logger.warn("Import failures: #{import_failures}") if example.exception && !import_failures.empty? + + user.remove_via_api! + ensure + Runtime::Feature.disable(:bulk_import_projects) + end + end +end diff --git a/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb b/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb index e47e5d22e5e..6a31d173440 100644 --- a/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb +++ b/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb @@ -37,7 +37,7 @@ module QA push.file_name = 'test.txt' push.file_content = "# This is a test project named #{@project.name}" push.commit_message = 'Add test.txt' - push.branch_name = 'new_branch' + push.branch_name = "new_branch_#{SecureRandom.hex(8)}" push.user = @user end end.to raise_error(QA::Support::Run::CommandError, /You are not allowed to push code to this project/) @@ -48,7 +48,7 @@ module QA Resource::File.fabricate_via_api! do |file| file.api_client = @user_api_client file.project = @project - file.branch = 'new_branch' + file.branch = "new_branch_#{SecureRandom.hex(8)}" file.commit_message = 'Add new file' file.name = 'test.txt' file.content = "New file" @@ -61,7 +61,7 @@ module QA Resource::Repository::Commit.fabricate_via_api! do |commit| commit.api_client = @user_api_client commit.project = @project - commit.branch = 'new_branch' + commit.branch = "new_branch_#{SecureRandom.hex(8)}" commit.start_branch = @project.default_branch commit.commit_message = 'Add new file' commit.add_files([ diff --git a/qa/qa/specs/features/api/3_create/gitaly/automatic_failover_and_recovery_spec.rb b/qa/qa/specs/features/api/3_create/gitaly/automatic_failover_and_recovery_spec.rb index 51927a30987..6a9be19efdd 100644 --- a/qa/qa/specs/features/api/3_create/gitaly/automatic_failover_and_recovery_spec.rb +++ b/qa/qa/specs/features/api/3_create/gitaly/automatic_failover_and_recovery_spec.rb @@ -49,7 +49,6 @@ module QA # for Gitaly to be ready for writes again praefect_manager.stop_primary_node praefect_manager.wait_for_primary_node_health_check_failure - praefect_manager.wait_for_gitaly_check Resource::Repository::Commit.fabricate_via_api! do |commit| commit.project = project diff --git a/qa/qa/specs/features/api/3_create/gitaly/praefect_connectivity_spec.rb b/qa/qa/specs/features/api/3_create/gitaly/praefect_connectivity_spec.rb new file mode 100644 index 00000000000..28469b99d04 --- /dev/null +++ b/qa/qa/specs/features/api/3_create/gitaly/praefect_connectivity_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Create' do + context 'Praefect connectivity commands', :orchestrated, :gitaly_cluster do + praefect_manager = Service::PraefectManager.new + + before do + praefect_manager.start_all_nodes + end + + context 'in a healthy environment' do + it 'confirms healthy connection to database', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349937' do + expect(praefect_manager.praefect_sql_ping_healthy?).to be true + end + + it 'confirms healthy connection to gitaly nodes', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349938' do + expect(praefect_manager.wait_for_dial_nodes_successful).to be true + end + end + + context 'in an unhealthy environment' do + it 'diagnoses unhealthy connection to database', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349939' do + praefect_manager.stop_node(praefect_manager.postgres) + expect(praefect_manager.praefect_sql_ping_healthy?).to be false + end + + it 'diagnoses connection issues to gitaly nodes', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349940' do + praefect_manager.stop_node(praefect_manager.primary_node) + praefect_manager.stop_node(praefect_manager.tertiary_node) + expect(praefect_manager.praefect_dial_nodes_status?(praefect_manager.primary_node, false)).to be true + expect(praefect_manager.praefect_dial_nodes_status?(praefect_manager.secondary_node)).to be true + expect(praefect_manager.praefect_dial_nodes_status?(praefect_manager.tertiary_node, false)).to be true + + praefect_manager.stop_node(praefect_manager.secondary_node) + expect(praefect_manager.praefect_dial_nodes_status?(praefect_manager.secondary_node, false)).to be true + end + end + end + end +end diff --git a/qa/qa/specs/features/api/3_create/gitaly/praefect_repo_sync_spec.rb b/qa/qa/specs/features/api/3_create/gitaly/praefect_repo_sync_spec.rb index cc49e408954..e27f37abedf 100644 --- a/qa/qa/specs/features/api/3_create/gitaly/praefect_repo_sync_spec.rb +++ b/qa/qa/specs/features/api/3_create/gitaly/praefect_repo_sync_spec.rb @@ -2,7 +2,7 @@ module QA RSpec.describe 'Create' do - context 'Praefect repository commands', :orchestrated, :gitaly_cluster, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/347415', type: :investigating } do + context 'Praefect repository commands', :orchestrated, :gitaly_cluster do let(:praefect_manager) { Service::PraefectManager.new } let(:repo1) { { "relative_path" => "@hashed/repo1.git", "storage" => "gitaly1", "virtual_storage" => "default" } } @@ -59,6 +59,18 @@ module QA untracked_repositories = praefect_manager.list_untracked_repositories expect(untracked_repositories).not_to include(repo1) end + + it 'allows admin to control the number of replicas of data', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347566' do + praefect_manager.track_repository_in_praefect(repo1['relative_path'], repo1['storage'], repo1['virtual_storage']) + + praefect_manager.set_replication_factor(repo1['relative_path'], repo1['virtual_storage'], 2) + replication_storages = praefect_manager.get_replication_storages(repo1['relative_path'], repo1['virtual_storage']) + expect(replication_storages).to have_attributes(size: 2) + + praefect_manager.set_replication_factor(repo1['relative_path'], repo1['virtual_storage'], 3) + replication_storages = praefect_manager.get_replication_storages(repo1['relative_path'], repo1['virtual_storage']) + expect(replication_storages).to eq(%w(gitaly1 gitaly2 gitaly3)) + end end end end diff --git a/qa/qa/specs/features/browser_ui/1_manage/group/bulk_import_group_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/group/gitlab_migration_group_spec.rb index 74125b092b8..a18e22f52f1 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/group/bulk_import_group_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/group/gitlab_migration_group_spec.rb @@ -1,9 +1,8 @@ # frozen_string_literal: true module QA - RSpec.describe 'Manage', :requires_admin do - describe 'Bulk group import' do - let!(:staging?) { Runtime::Scenario.gitlab_address.include?('staging.gitlab.com') } + describe 'Manage', :requires_admin do + describe 'Gitlab migration' do let!(:admin_api_client) { Runtime::API::Client.as_admin } let!(:user) do Resource::User.fabricate_via_api! do |usr| @@ -32,7 +31,7 @@ module QA Resource::BulkImportGroup.init do |group| group.api_client = api_client group.sandbox = sandbox - group.source_group_path = source_group.path + group.source_group = source_group end end diff --git a/qa/qa/specs/features/browser_ui/1_manage/login/register_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/login/register_spec.rb index 16f8df5a90d..098c0b3ba63 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/login/register_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/login/register_spec.rb @@ -64,6 +64,7 @@ module QA Page::Profile::Accounts::Show.perform do |show| show.delete_account(user.password) end + Support::Waiter.wait_until { !user.exists? } end it 'allows recreating with same credentials', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347868' do @@ -83,7 +84,7 @@ module QA end after do - @recreated_user.remove_via_api! + @recreated_user&.remove_via_api! end def admin_api_client @@ -117,11 +118,12 @@ module QA Flow::Login.sign_in(as: @user, skip_page_validation: true) - Page::Registration::Welcome.perform(&:click_get_started_button_if_available) + Flow::UserOnboarding.onboard_user - Page::Main::Menu.perform do |menu| - expect(menu).to have_personal_area - end + # In development env and .com the user is asked to create a group and a project which can be skipped for + # the purpose of this test + Runtime::Browser.visit(:gitlab, Page::Dashboard::Welcome) + Page::Main::Menu.perform(&:has_personal_area?) end after do diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb index 6d09c8b1316..895027a588d 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb @@ -1,7 +1,11 @@ # frozen_string_literal: true module QA - RSpec.describe 'Manage', :requires_admin do + RSpec.describe 'Manage', :requires_admin, quarantine: { + issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/350598', + type: :needs_update, + only: { subdomain: :staging } + } do describe 'Add project member' do before do Runtime::Feature.enable(:invite_members_group_modal) diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb index 7f40818da03..0063ce2613a 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb @@ -2,7 +2,7 @@ module QA RSpec.describe 'Manage', :smoke do - describe 'Project', :requires_admin do + describe 'Project' do shared_examples 'successful project creation' do it 'creates a new project' do Page::Project::Show.perform do |project_page| @@ -17,7 +17,6 @@ module QA end before do - Runtime::Feature.enable(:paginatable_namespace_drop_down_for_project_creation) Flow::Login.sign_in project end diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/invite_group_to_project_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/invite_group_to_project_spec.rb new file mode 100644 index 00000000000..6997447411a --- /dev/null +++ b/qa/qa/specs/features/browser_ui/1_manage/project/invite_group_to_project_spec.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module QA + # Tagging with issue for a transient invite group modal search bug, but does not require quarantine at this time + RSpec.describe 'Manage', :requires_admin, :transient, issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/349379' do + describe 'Invite group' do + shared_examples 'invites group to project' do + it 'verifies group is added and members can access project with correct access level' do + Page::Project::Menu.perform(&:click_members) + Page::Project::Members.perform do |project_members| + project_members.invite_group(group.path, 'Developer') + + expect(project_members).to have_group(group.path) + end + + Flow::Login.sign_in(as: @user) + + Page::Dashboard::Projects.perform do |projects| + expect(projects).to have_project_with_access_role(project.name, 'Developer') + end + + project.visit! + + Page::Project::Show.perform do |project_page| + expect(project_page).to have_name(project.name) + end + end + end + + before(:context) do + Runtime::Feature.enable(:invite_members_group_modal) + @user = Resource::User.fabricate_or_use(Runtime::Env.gitlab_qa_username_1, Runtime::Env.gitlab_qa_password_1) + end + + before do + Flow::Login.sign_in + group.add_member(@user, Resource::Members::AccessLevel::MAINTAINER) + project.visit! + end + + context 'to personal namespace project', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349223' do + let(:group) do + Resource::Group.fabricate_via_api! do |group| + group.path = "group-for-personal-project-#{SecureRandom.hex(8)}" + end + end + + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'personal-namespace-project' + project.personal_namespace = Runtime::User.username + project.visibility = :private + project.description = 'test personal namespace project' + end + end + + it_behaves_like 'invites group to project' + end + + context 'to group project', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349340' do + let(:group) do + Resource::Group.fabricate_via_api! do |group| + group.path = "group-for-group-project-#{SecureRandom.hex(8)}" + end + end + + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'group-project' + project.visibility = :private + project.description = 'test group project' + end + end + + it_behaves_like 'invites group to project' + end + + after do + project&.remove_via_api! + group&.remove_via_api! + end + + after(:context) do + Runtime::Feature.disable(:invite_members_group_modal) + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/project_access_token_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/project_access_token_spec.rb index be8567ee0b6..c2bd61155b1 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/project/project_access_token_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/project/project_access_token_spec.rb @@ -9,7 +9,7 @@ module QA expect(project_access_token.token).not_to be_nil project_access_token.revoke_via_ui! - expect(page).to have_text("Revoked project access token #{project_access_token.name}!") + expect(page).to have_text("Revoked access token #{project_access_token.name}!") end after do diff --git a/qa/qa/specs/features/browser_ui/1_manage/user/follow_user_activity_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/user/follow_user_activity_spec.rb index 43100929acd..87b51edef08 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/user/follow_user_activity_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/user/follow_user_activity_spec.rb @@ -19,7 +19,7 @@ module QA group = QA::Resource::Group.fabricate_via_api! do |group| group.path = "group_for_follow_user_activity_#{SecureRandom.hex(8)}" end - group.add_member(user) + group.add_member(user, Resource::Members::AccessLevel::MAINTAINER) group end diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb index d198d79c5fe..b0c6d01e8ca 100644 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb @@ -2,7 +2,11 @@ module QA RSpec.describe 'Create' do - describe 'Merge request creation from fork' do + describe 'Merge request creation from fork', quarantine: { + only: { subdomain: %i[canary production] }, + issue: "https://gitlab.com/gitlab-org/gitlab/-/issues/343801", + type: :investigation + } do let(:merge_request) do Resource::MergeRequestFromFork.fabricate_via_browser_ui! do |merge_request| merge_request.fork_branch = 'feature-branch' diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/view_merge_request_merge_ref_diff_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/view_merge_request_merge_ref_diff_spec.rb deleted file mode 100644 index 0785b32b225..00000000000 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/view_merge_request_merge_ref_diff_spec.rb +++ /dev/null @@ -1,90 +0,0 @@ -# frozen_string_literal: true - -module QA - RSpec.describe 'Create', :requires_admin, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/261793', type: :investigating } do - describe 'View merge request merge-ref diff' do - let(:project) do - Resource::Project.fabricate_via_api! do |project| - project.name = 'merge-ref-diff' - end - end - - let(:merge_request) do - Resource::MergeRequest.fabricate_via_api! do |merge_request| - merge_request.project = project - merge_request.title = 'This is a merge request' - merge_request.description = '... for viewing merge-ref and merge-base diffs' - merge_request.file_content = 'This exists on the source branch only' - end - end - - let(:new_file_name) { "added_file-#{SecureRandom.hex(8)}.txt" } - - context 'when the feature flag default_merge_ref_for_diffs is enabled' do - before do - Runtime::Feature.enable('default_merge_ref_for_diffs', project: project) - - commit_to_branch(merge_request.target_branch, new_file_name) - commit_to_branch(merge_request.source_branch, new_file_name) - - Flow::Login.sign_in - - merge_request.visit! - end - - it 'views the merge-ref diff by default', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347651' do - Page::MergeRequest::Show.perform do |mr_page| - mr_page.click_diffs_tab - mr_page.click_target_version_dropdown - - expect(mr_page.version_dropdown_content).to include("#{project.default_branch} (HEAD)") - expect(mr_page.version_dropdown_content).not_to include("#{project.default_branch} (base)") - expect(mr_page).to have_file(merge_request.file_name) - expect(mr_page).not_to have_file(new_file_name) - end - end - end - - context 'when the feature flag default_merge_ref_for_diffs is disabled' do - before do - Runtime::Feature.disable('default_merge_ref_for_diffs', project: project) - - commit_to_branch(merge_request.target_branch, new_file_name) - commit_to_branch(merge_request.source_branch, new_file_name) - - Flow::Login.sign_in - - merge_request.visit! - end - - it 'views the merge-base diff by default', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347650' do - Page::MergeRequest::Show.perform do |mr_page| - mr_page.click_diffs_tab - mr_page.click_target_version_dropdown - - expect(mr_page.version_dropdown_content).to include("#{project.default_branch} (HEAD)") - expect(mr_page.version_dropdown_content).to include("#{project.default_branch} (base)") - expect(mr_page).to have_file(merge_request.file_name) - expect(mr_page).to have_file(new_file_name) - end - end - end - - def commit_to_branch(branch, file) - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.project = merge_request.project - commit.branch = branch - commit.commit_message = "Add new file on #{branch}" - commit.add_files( - [ - { - file_path: file, - content: "This exists on source and target branches" - } - ] - ) - end - end - end - end -end diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb index a98925eab98..0bd470fcb77 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb @@ -24,8 +24,6 @@ module QA proj.initialize_with_readme = true end - Runtime::Feature.enable(:delete_branch_confirmation_modals, project: project) - master_branch = project.default_branch Git::Repository.perform do |repository| diff --git a/qa/qa/specs/features/browser_ui/3_create/wiki/content_editor_spec.rb b/qa/qa/specs/features/browser_ui/3_create/wiki/content_editor_spec.rb index a063acbe146..67eee66b3d6 100644 --- a/qa/qa/specs/features/browser_ui/3_create/wiki/content_editor_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/wiki/content_editor_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Create', :requires_admin do # remove :requires_admin once the ff is enabled by default in https://gitlab.com/gitlab-org/gitlab/-/issues/345398 + RSpec.describe 'Create', :requires_admin, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/350220', type: :investigating } do # remove :requires_admin once the ff is enabled by default in https://gitlab.com/gitlab-org/gitlab/-/issues/345398 context 'Content Editor' do let(:initial_wiki) { Resource::Wiki::ProjectPage.fabricate_via_api! } let(:page_title) { 'Content Editor Page' } diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_lint_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_lint_spec.rb new file mode 100644 index 00000000000..8f3284662d7 --- /dev/null +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_lint_spec.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Verify' do + describe 'Pipeline editor' do + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'pipeline-editor-project' + end + end + + let!(:commit) do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files( + [ + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + stages: + - stage1 + - stage2 + + job1: + stage: stage1 + script: echo 'Done.' + + job2: + stage: stage2 + script: echo 'Done.' + YAML + } + ] + ) + end + end + + before do + Flow::Login.sign_in + project.visit! + Page::Project::Menu.perform(&:go_to_pipeline_editor) + end + + after do + project&.remove_via_api! + end + + context 'when CI has valid syntax' do + it 'shows valid validations', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349128' do + Page::Project::PipelineEditor::Show.perform do |show| + aggregate_failures do + expect(show.ci_syntax_validate_message).to have_content('CI configuration is valid') + + show.go_to_visualize_tab + { stage1: 'job1', stage2: 'job2' }.each_pair do |stage, job| + expect(show).to have_stage(stage), "Pipeline graph does not have stage #{stage}." + expect(show).to have_job(job), "Pipeline graph does not have job #{job}." + end + + show.go_to_lint_tab + expect(show.tab_alert_message).to have_content('Syntax is correct') + + show.go_to_view_merged_yaml_tab + expect(show).to have_source_editor + end + end + end + end + + context 'when CI has invalid syntax' do + it 'shows invalid validations', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349129' do + invalid_msg = 'syntax is invalid' + + Page::Project::PipelineEditor::Show.perform do |show| + show.write_to_editor(SecureRandom.hex(10)) + + aggregate_failures do + show.go_to_visualize_tab + expect(show.tab_alert_message).to have_content(invalid_msg) + + show.go_to_lint_tab + expect(show.tab_alert_message).to have_content('Syntax is incorrect') + + show.go_to_view_merged_yaml_tab + expect(show.tab_alert_message).to have_content(invalid_msg) + + expect(show.ci_syntax_validate_message).to have_content('CI configuration is invalid') + end + end + end + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/update_ci_file_with_pipeline_editor_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/update_ci_file_with_pipeline_editor_spec.rb new file mode 100644 index 00000000000..00c5d4c74d4 --- /dev/null +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/update_ci_file_with_pipeline_editor_spec.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Verify' do + describe 'Update CI file with pipeline editor' do + let(:random_test_string) { SecureRandom.hex(10) } + + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'pipeline-editor-project' + end + end + + let!(:runner) do + Resource::Runner.fabricate_via_api! do |runner| + runner.project = project + runner.name = random_test_string + runner.tags = [random_test_string] + end + end + + let!(:commit) do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files( + [ + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + test_job: + tags: ['#{random_test_string}'] + script: + - echo "Simple test!" + YAML + } + ] + ) + end + end + + before do + Flow::Login.sign_in + project.visit! + Support::Waiter.wait_until { !project.pipelines.empty? && project.pipelines.first[:status] == 'success' } + Page::Project::Menu.perform(&:go_to_pipeline_editor) + end + + after do + [runner, project].each(&:remove_via_api!) + end + + it 'creates new pipeline and target branch', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349005' do + Page::Project::PipelineEditor::Show.perform do |show| + show.write_to_editor(random_test_string) + show.set_target_branch(random_test_string) + show.submit_changes + + Support::Waiter.wait_until { project.pipelines.size > 1 } + + aggregate_failures do + expect(show.target_branch_name).to eq(random_test_string) + expect(show.current_branch).to eq(random_test_string) + expect(show.editing_content).to have_content(random_test_string) + expect { show.pipeline_id }.to eventually_eq(project.pipelines.pluck(:id).max).within(max_duration: 60, sleep_interval: 3) + end + end + + expect(project).to have_branch(random_test_string) + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/5_package/container_registry/online_garbage_collection_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry/online_garbage_collection_spec.rb index e8d936e67b1..56e3ec82388 100644 --- a/qa/qa/specs/features/browser_ui/5_package/container_registry/online_garbage_collection_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/container_registry/online_garbage_collection_spec.rb @@ -2,8 +2,7 @@ module QA RSpec.describe 'Package' do - # TODO: Remove :requires_admin when the `Runtime::Feature.enable` method call is removed - describe 'Container Registry Online Garbage Collection', :registry_gc, :requires_admin, only: { subdomain: %i[pre] } do + describe 'Container Registry Online Garbage Collection', :registry_gc, only: { subdomain: %i[pre] } do let(:group) { Resource::Group.fabricate_via_api! } let(:imported_project) do @@ -65,8 +64,6 @@ module QA end before do - Runtime::Feature.enable(:paginatable_namespace_drop_down_for_project_creation) - Flow::Login.sign_in imported_project diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb index c58cdec622d..70b31c1beca 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Package Registry', :orchestrated, :packages, :reliable, :object_storage do + RSpec.describe 'Package Registry', :orchestrated, :packages, :object_storage do describe 'npm instance level endpoint' do using RSpec::Parameterized::TableSyntax include Runtime::Fixtures diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb index cec902e073a..e25a742493b 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Package Registry', :orchestrated, :packages, :reliable, :object_storage do + RSpec.describe 'Package Registry', :orchestrated, :packages, :object_storage do describe 'npm project level endpoint' do using RSpec::Parameterized::TableSyntax include Runtime::Fixtures diff --git a/qa/qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_independent_relationship_spec.rb b/qa/qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_independent_relationship_spec.rb index 74a81ff429d..ef0c8d35c37 100644 --- a/qa/qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_independent_relationship_spec.rb +++ b/qa/qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_independent_relationship_spec.rb @@ -13,7 +13,7 @@ module QA Resource::Runner.fabricate_via_api! do |runner| runner.project = project runner.name = project.name - runner.tags = ["#{project.name}"] + runner.tags = [project.name] end end diff --git a/qa/qa/specs/features/browser_ui/8_monitor/.gitkeep b/qa/qa/specs/features/browser_ui/8_monitor/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/qa/qa/specs/features/browser_ui/8_monitor/.gitkeep diff --git a/qa/qa/specs/features/browser_ui/8_monitor/all_monitor_core_features_spec.rb b/qa/qa/specs/features/browser_ui/8_monitor/all_monitor_core_features_spec.rb deleted file mode 100644 index c13d2d2dddf..00000000000 --- a/qa/qa/specs/features/browser_ui/8_monitor/all_monitor_core_features_spec.rb +++ /dev/null @@ -1,142 +0,0 @@ -# frozen_string_literal: true -require_relative 'cluster_with_prometheus' - -module QA - RSpec.describe 'Monitor', :orchestrated, :kubernetes, :requires_admin, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/241448', type: :investigating } do - include_context "cluster with Prometheus installed" - - before do - Flow::Login.sign_in_unless_signed_in - @project.visit! - end - - it 'configures custom metrics', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348082' do - verify_add_custom_metric - verify_edit_custom_metric - verify_delete_custom_metric - end - - it 'duplicates to create dashboard to custom', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348070' do - Page::Project::Menu.perform(&:go_to_monitor_metrics) - - Page::Project::Monitor::Metrics::Show.perform do |on_dashboard| - on_dashboard.duplicate_dashboard - - expect(on_dashboard).to have_metrics - expect(on_dashboard).to have_edit_dashboard_enabled - end - end - - it 'verifies data on filtered deployed environment', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348071' do - Page::Project::Menu.perform(&:go_to_monitor_metrics) - - Page::Project::Monitor::Metrics::Show.perform do |on_dashboard| - on_dashboard.filter_environment - - expect(on_dashboard).to have_metrics - end - end - - it 'filters using the quick range', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348083' do - Page::Project::Menu.perform(&:go_to_monitor_metrics) - - Page::Project::Monitor::Metrics::Show.perform do |on_dashboard| - on_dashboard.show_last('30 minutes') - expect(on_dashboard).to have_metrics - - on_dashboard.show_last('3 hours') - expect(on_dashboard).to have_metrics - - on_dashboard.show_last('1 day') - expect(on_dashboard).to have_metrics - end - end - - it 'observes cluster health graph', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348074' do - Page::Project::Menu.perform(&:go_to_infrastructure_kubernetes) - - Page::Project::Infrastructure::Kubernetes::Index.perform do |cluster_list| - cluster_list.click_on_cluster(@cluster) - end - - Page::Project::Infrastructure::Kubernetes::Show.perform do |cluster_panel| - cluster_panel.open_health - cluster_panel.wait_for_cluster_health - end - end - - it 'uses templating variables for metrics dashboards', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347636' do - templating_dashboard_yml = Pathname - .new(__dir__) - .join('../../../../fixtures/metrics_dashboards/templating.yml') - - Resource::Repository::ProjectPush.fabricate! do |push| - push.project = @project - push.file_name = '.gitlab/dashboards/templating.yml' - push.file_content = File.read(templating_dashboard_yml) - push.commit_message = 'Add templating in dashboard file' - push.new_branch = false - end - - Page::Project::Menu.perform(&:go_to_monitor_metrics) - - Page::Project::Monitor::Metrics::Show.perform do |dashboard| - dashboard.select_dashboard('templating.yml') - - expect(dashboard).to have_template_metric('CPU usage GitLab Runner') - expect(dashboard).to have_template_metric('Memory usage Postgresql') - expect(dashboard).to have_templating_variable('GitLab Runner') - expect(dashboard).to have_templating_variable('Postgresql') - end - end - - private - - def verify_add_custom_metric - Page::Project::Menu.perform(&:go_to_integrations_settings) - Page::Project::Settings::Integrations.perform(&:click_on_prometheus_integration) - - Page::Project::Settings::Services::Prometheus.perform do |metrics_panel| - metrics_panel.click_on_new_metric - metrics_panel.add_custom_metric - end - - Page::Project::Menu.perform(&:go_to_monitor_metrics) - - Page::Project::Monitor::Metrics::Show.perform do |on_dashboard| - expect(on_dashboard).to have_custom_metric('HTTP Requests Total') - end - end - - def verify_edit_custom_metric - Page::Project::Menu.perform(&:go_to_integrations_settings) - Page::Project::Settings::Integrations.perform(&:click_on_prometheus_integration) - Page::Project::Settings::Services::Prometheus.perform do |metrics_panel| - metrics_panel.click_on_custom_metric('Business / HTTP Requests Total (req/sec)') - metrics_panel.edit_custom_metric - end - - Page::Project::Menu.perform(&:go_to_monitor_metrics) - - Page::Project::Monitor::Metrics::Show.perform do |on_dashboard| - expect(on_dashboard).to have_custom_metric('Throughput') - end - end - - def verify_delete_custom_metric - Page::Project::Menu.perform(&:go_to_integrations_settings) - Page::Project::Settings::Integrations.perform(&:click_on_prometheus_integration) - - Page::Project::Settings::Services::Prometheus.perform do |metrics_panel| - metrics_panel.click_on_custom_metric('Business / Throughput (req/sec)') - metrics_panel.delete_custom_metric - end - - Page::Project::Menu.perform(&:go_to_monitor_metrics) - - Page::Project::Monitor::Metrics::Show.perform do |on_dashboard| - expect(on_dashboard).not_to have_custom_metric('Throughput') - end - end - end -end diff --git a/qa/qa/specs/features/browser_ui/8_monitor/cluster_with_prometheus.rb b/qa/qa/specs/features/browser_ui/8_monitor/cluster_with_prometheus.rb deleted file mode 100644 index 19e49400d5e..00000000000 --- a/qa/qa/specs/features/browser_ui/8_monitor/cluster_with_prometheus.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true - -module QA - RSpec.shared_context "cluster with Prometheus installed" do - before :all do - @cluster = Service::KubernetesCluster.new(provider_class: Service::ClusterProvider::K3s).create! - @project = Resource::Project.fabricate_via_api! do |project| - project.name = 'monitoring-project' - project.auto_devops_enabled = true - project.template_name = 'express' - end - - deploy_project_with_prometheus - end - - def deploy_project_with_prometheus - %w[ - CODE_QUALITY_DISABLED TEST_DISABLED LICENSE_MANAGEMENT_DISABLED - SAST_DISABLED DAST_DISABLED DEPENDENCY_SCANNING_DISABLED - CONTAINER_SCANNING_DISABLED BROWSER_PERFORMANCE_DISABLED SECRET_DETECTION_DISABLED - ].each do |key| - Resource::CiVariable.fabricate_via_api! do |resource| - resource.project = @project - resource.key = key - resource.value = '1' - resource.masked = false - end - end - - Flow::Login.sign_in - - Resource::KubernetesCluster::ProjectCluster.fabricate! do |cluster_settings| - cluster_settings.project = @project - cluster_settings.cluster = @cluster - cluster_settings.install_runner = true - cluster_settings.install_ingress = true - cluster_settings.install_prometheus = true - end - - Resource::Pipeline.fabricate_via_api! do |pipeline| - pipeline.project = @project - end.visit! - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('build') - end - Page::Project::Job::Show.perform do |job| - expect(job).to be_successful(timeout: 600) - - job.click_element(:pipeline_path) - end - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('production') - end - Page::Project::Job::Show.perform do |job| - expect(job).to be_successful(timeout: 1200) - - job.click_element(:pipeline_path) - end - end - - after :all do - @cluster&.remove! - end - end -end diff --git a/qa/qa/specs/helpers/quarantine.rb b/qa/qa/specs/helpers/quarantine.rb index 49d91fc87cd..738c99efb28 100644 --- a/qa/qa/specs/helpers/quarantine.rb +++ b/qa/qa/specs/helpers/quarantine.rb @@ -11,9 +11,9 @@ module QA extend self # Skip tests in quarantine unless we explicitly focus on them. - def skip_or_run_quarantined_tests_or_contexts(filters, example) + def skip_or_run_quarantined_tests_or_contexts(example) if filters.key?(:quarantine) - included_filters = filters_other_than_quarantine(filters) + included_filters = filters_other_than_quarantine # If :quarantine is focused, skip the test/context unless its metadata # includes quarantine and any other filters @@ -29,18 +29,17 @@ module QA elsif example.metadata.key?(:quarantine) quarantine_tag = example.metadata[:quarantine] - if quarantine_tag.is_a?(Hash) && quarantine_tag&.key?(:only) && !ContextSelector.context_matches?(quarantine_tag[:only]) - # If the :quarantine hash contains :only, we respect that. - # For instance `quarantine: { only: { subdomain: :staging } }` will only quarantine the test when it runs against staging. - return - end + # If the :quarantine hash contains :only, we respect that. + # For instance `quarantine: { only: { subdomain: :staging } }` + # will only quarantine the test when it runs against staging. + return if quarantined_different_context?(quarantine_tag) example.metadata[:skip] = quarantine_message(quarantine_tag) end end - def filters_other_than_quarantine(filter) - filter.reject { |key, _| key == :quarantine } + def filters_other_than_quarantine + filters.reject { |key, _| key == :quarantine } end def quarantine_message(quarantine_tag) @@ -70,6 +69,14 @@ module QA (metadata.keys & included_filters.keys).empty? end + + def quarantined_different_context?(quarantine) + quarantine.is_a?(Hash) && quarantine.key?(:only) && !ContextSelector.context_matches?(quarantine[:only]) + end + + def filters + @filters ||= ::RSpec.configuration.inclusion_filter.rules + end end end end diff --git a/qa/qa/specs/runner.rb b/qa/qa/specs/runner.rb index d7d64834e7a..2c9e302fc56 100644 --- a/qa/qa/specs/runner.rb +++ b/qa/qa/specs/runner.rb @@ -9,6 +9,7 @@ module QA attr_accessor :tty, :tags, :options DEFAULT_TEST_PATH_ARGS = ['--', File.expand_path('./features', __dir__)].freeze + DEFAULT_STD_ARGS = [$stderr, $stdout].freeze def initialize @tty = false @@ -19,13 +20,11 @@ module QA def paths_from_knapsack allocator = Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator - QA::Runtime::Logger.info '' + QA::Runtime::Logger.info '==== Knapsack specs to execute =====' QA::Runtime::Logger.info 'Report specs:' QA::Runtime::Logger.info allocator.report_node_tests.join(', ') - QA::Runtime::Logger.info '' QA::Runtime::Logger.info 'Leftover specs:' QA::Runtime::Logger.info allocator.leftover_node_tests.join(', ') - QA::Runtime::Logger.info '' ['--', allocator.node_tests] end @@ -70,8 +69,15 @@ module QA ParallelRunner.run(args.flatten) elsif Runtime::Scenario.attributes[:loop] LoopRunner.run(args.flatten) + elsif Runtime::Scenario.attributes[:count_examples_only] + args.unshift('--dry-run') + out = StringIO.new + RSpec::Core::Runner.run(args.flatten, $stderr, out).tap do |status| + abort if status.nonzero? + end + $stdout.puts out.string.match(/(\d+) examples,/)[1] else - RSpec::Core::Runner.run(args.flatten, $stderr, $stdout).tap do |status| + RSpec::Core::Runner.run(args.flatten, *DEFAULT_STD_ARGS).tap do |status| abort if status.nonzero? end end diff --git a/qa/qa/support/formatters/quarantine_formatter.rb b/qa/qa/support/formatters/quarantine_formatter.rb index c5d16988dbd..4e4da99749c 100644 --- a/qa/qa/support/formatters/quarantine_formatter.rb +++ b/qa/qa/support/formatters/quarantine_formatter.rb @@ -18,7 +18,7 @@ module QA def example_group_started(example_group_notification) group = example_group_notification.group - skip_or_run_quarantined_tests_or_contexts(filters, group) + skip_or_run_quarantined_tests_or_contexts(group) end # Starts example @@ -28,13 +28,7 @@ module QA example = example_notification.example # if skip propagated from example_group, do not reset skip metadata - skip_or_run_quarantined_tests_or_contexts(filters, example) unless example.metadata[:skip] - end - - private - - def filters - @filters ||= ::RSpec.configuration.inclusion_filter.rules + skip_or_run_quarantined_tests_or_contexts(example) unless example.metadata[:skip] end end end diff --git a/qa/qa/support/formatters/test_stats_formatter.rb b/qa/qa/support/formatters/test_stats_formatter.rb index 6f6291b5856..7678cb8406c 100644 --- a/qa/qa/support/formatters/test_stats_formatter.rb +++ b/qa/qa/support/formatters/test_stats_formatter.rb @@ -84,7 +84,8 @@ module QA retry_attempts: example.metadata[:retry_attempts] || 0, job_url: QA::Runtime::Env.ci_job_url, pipeline_url: env('CI_PIPELINE_URL'), - pipeline_id: env('CI_PIPELINE_ID') + pipeline_id: env('CI_PIPELINE_ID'), + testcase: example.metadata[:testcase] } } rescue StandardError => e @@ -124,11 +125,11 @@ module QA @merge_request ||= (!!env('CI_MERGE_REQUEST_IID') || !!env('TOP_UPSTREAM_MERGE_REQUEST_IID')).to_s end - # Test run type from staging, canary, preprod or production env + # Test run type from staging (`gstg`, `gstg-cny`, `gstg-ref`), canary, preprod or production env # # @return [String, nil] def run_type - return unless %w[staging canary preprod production].include?(project_name) + return unless %w[staging staging-canary staging-ref canary preprod production].include?(project_name) @run_type ||= begin test_subset = if env('NO_ADMIN') == 'true' diff --git a/qa/qa/support/matchers/eventually_matcher.rb b/qa/qa/support/matchers/eventually_matcher.rb index dedef8e6b98..2fb5249d9af 100644 --- a/qa/qa/support/matchers/eventually_matcher.rb +++ b/qa/qa/support/matchers/eventually_matcher.rb @@ -28,27 +28,21 @@ module QA RSpec::Matchers.define(:"eventually_#{op}") do |*expected| chain(:within) do |kwargs = {}| @retry_args = kwargs - @retry_args[:sleep_interval] = 0.5 unless @retry_args[:sleep_interval] + @retry_args[:sleep_interval] = 0.5 unless kwargs[:sleep_interval] end - def supports_block_expectations? - true - end + description { "eventually #{operator_msg}: #{expected_formatted}" } match { |actual| wait_and_check(actual, :default_expectation) } match_when_negated { |actual| wait_and_check(actual, :when_negated_expectation) } - description do - "eventually #{operator_msg} #{expected.inspect}" - end + failure_message { fail_message } - failure_message do - "#{e}:\nexpected to #{description}, last attempt was #{@result.nil? ? 'nil' : @result}" - end + failure_message_when_negated { fail_message(negate: true) } - failure_message_when_negated do - "#{e}:\nexpected not to #{description}, last attempt was #{@result.nil? ? 'nil' : @result}" + def supports_block_expectations? + true end # Execute rspec expectation within retrier @@ -60,9 +54,9 @@ module QA attempt = 0 QA::Runtime::Logger.debug( - "Running eventually matcher with '#{operator_msg}' operator with: #{@retry_args}" + "Running eventually matcher with '#{operator_msg}' operator with: '#{retry_args}' arguments" ) - QA::Support::Retrier.retry_until(**@retry_args, log: false) do + QA::Support::Retrier.retry_until(**retry_args, log: false) do QA::Runtime::Logger.debug("evaluating expectation, attempt: #{attempt += 1}") public_send(expectation_name, actual) @@ -132,6 +126,44 @@ module QA [operator, expected] end end + + # Custom retry arguments + # + # @return [Hash] + def retry_args + @retry_args ||= { sleep_interval: 0.5 } + end + + # Custom failure message + # + # @param [Boolean] negate + # @return [String] + def fail_message(negate: false) + "#{e}:\n\nexpected #{negate ? 'not ' : ''}to #{description}\n\n"\ + "last attempt was: #{@result.nil? ? 'nil' : actual_formatted}\n\n"\ + "Diff:#{diff}" + end + + # Formatted expect + # + # @return [String] + def expected_formatted + RSpec::Support::ObjectFormatter.format(expected) + end + + # Formatted actual result + # + # @return [String] + def actual_formatted + RSpec::Support::ObjectFormatter.format(@result) + end + + # Object diff + # + # @return [String] + def diff + RSpec::Support::Differ.new(color: true).diff(@result, expected) + end end end end diff --git a/qa/qa/support/matchers/have_matcher.rb b/qa/qa/support/matchers/have_matcher.rb index 47d2d246460..a90d2df96ae 100644 --- a/qa/qa/support/matchers/have_matcher.rb +++ b/qa/qa/support/matchers/have_matcher.rb @@ -5,6 +5,7 @@ module QA module Matchers module HaveMatcher PREDICATE_TARGETS = %w[ + auto_devops_container element file_content assignee @@ -17,6 +18,8 @@ module QA package pipeline related_issue_item + sast_status + security_configuration_history_link snippet_description tag label diff --git a/qa/qa/support/page_error_checker.rb b/qa/qa/support/page_error_checker.rb new file mode 100644 index 00000000000..5d16245b4cd --- /dev/null +++ b/qa/qa/support/page_error_checker.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module QA + module Support + class PageErrorChecker + class << self + def report!(page, error_code) + report = if QA::Runtime::Env.browser == :chrome + return_chrome_errors(page, error_code) + else + status_code_report(error_code) + end + + raise "#{report}\n\n"\ + "Path: #{page.current_path}" + end + + def return_chrome_errors(page, error_code) + severe_errors = logs(page).select { |log| log.level == 'SEVERE' } + if severe_errors.none? + status_code_report(error_code) + else + "There #{severe_errors.count == 1 ? 'was' : 'were'} #{severe_errors.count} "\ + "SEVERE level error#{severe_errors.count == 1 ? '' : 's'}:\n\n#{error_report_for(severe_errors)}" + end + end + + def status_code_report(error_code) + "Status code #{error_code} found" + end + + def check_page_for_error_code(page) + error_code = 0 + # Test for 404 img alt + error_code = 404 if Nokogiri::HTML.parse(page.html).xpath("//img").map { |t| t[:alt] }.first.eql?('404') + + # 500 error page in header surrounded by newlines, try to match + five_hundred_test = Nokogiri::HTML.parse(page.html).xpath("//h1").map.first + unless five_hundred_test.nil? + error_code = 500 if five_hundred_test.text.include?('500') + end + # GDK shows backtrace rather than error page + error_code = 500 if Nokogiri::HTML.parse(page.html).xpath("//body//section").map { |t| t[:class] }.first.eql?('backtrace') + + unless error_code == 0 + report!(page, error_code) + end + end + + def error_report_for(logs) + logs + .map(&:message) + .map { |message| message.gsub('\\n', "\n") } + end + + def logs(page) + page.driver.browser.manage.logs.get(:browser) + end + end + end + end +end diff --git a/qa/qa/support/wait_for_requests.rb b/qa/qa/support/wait_for_requests.rb index 5109f51d4d7..16af4bae521 100644 --- a/qa/qa/support/wait_for_requests.rb +++ b/qa/qa/support/wait_for_requests.rb @@ -7,7 +7,12 @@ module QA DEFAULT_MAX_WAIT_TIME = 60 - def wait_for_requests(skip_finished_loading_check: false) + def wait_for_requests(skip_finished_loading_check: false, skip_resp_code_check: false) + # We have tests that use 404 pages, allow them to skip this check + unless skip_resp_code_check + QA::Support::PageErrorChecker.check_page_for_error_code(Capybara.page) + end + Waiter.wait_until(log: false) do finished_all_ajax_requests? && (!skip_finished_loading_check ? finished_loading?(wait: 1) : true) end diff --git a/qa/qa/tools/delete_projects.rb b/qa/qa/tools/delete_projects.rb index 240901eea6f..1f550f035d1 100644 --- a/qa/qa/tools/delete_projects.rb +++ b/qa/qa/tools/delete_projects.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_relative '../../qa' - # This script deletes all projects directly under a group specified by ENV['TOP_LEVEL_GROUP_NAME'] # Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS # Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group') diff --git a/qa/qa/tools/delete_subgroups.rb b/qa/qa/tools/delete_subgroups.rb index bc905fdeadd..11b45365d4c 100644 --- a/qa/qa/tools/delete_subgroups.rb +++ b/qa/qa/tools/delete_subgroups.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_relative '../../qa' - # This script deletes all subgroups of a group specified by ENV['TOP_LEVEL_GROUP_NAME'] # Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS # Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group') diff --git a/qa/qa/tools/delete_test_resources.rb b/qa/qa/tools/delete_test_resources.rb new file mode 100644 index 00000000000..917cb2fa992 --- /dev/null +++ b/qa/qa/tools/delete_test_resources.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +# This script reads from test_resources.txt file to collect data about resources to delete +# Deletes all deletable resources that E2E tests created +# Resource type: Sandbox, User, Fork and RSpec::Mocks::Double are not included +# +# Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS +# When in CI also requires: QA_TEST_RESOURCES_FILE_PATTERN +# Run `rake delete_test_resources[<file_pattern>]` + +module QA + module Tools + class DeleteTestResources + include Support::API + + def initialize(file_pattern = nil) + raise ArgumentError, "Please provide GITLAB_ADDRESS" unless ENV['GITLAB_ADDRESS'] + raise ArgumentError, "Please provide GITLAB_QA_ACCESS_TOKEN" unless ENV['GITLAB_QA_ACCESS_TOKEN'] + + @api_client = Runtime::API::Client.new(ENV['GITLAB_ADDRESS'], personal_access_token: ENV['GITLAB_QA_ACCESS_TOKEN']) + @file_pattern = file_pattern + end + + def run + puts 'Deleting test created resources...' + + if Runtime::Env.running_in_ci? + raise ArgumentError, 'Please provide QA_TEST_RESOURCES_FILE_PATTERN' unless ENV['QA_TEST_RESOURCES_FILE_PATTERN'] + + Dir.glob(@file_pattern).each do |file| + delete_resources(load_file(file)) + end + else + file = Runtime::Env.test_resources_created_filepath + raise ArgumentError, "'#{file}' either does not exist or empty." if !File.exist?(file) || File.zero?(file) + + delete_resources(load_file(file)) + end + + puts "\nDone" + end + + private + + def load_file(json) + JSON.parse(File.read(json)) + end + + def delete_resources(resources) + failures = [] + + resources.each_key do |type| + next if resources[type].empty? + + resources[type].each do |resource| + next if resource_not_found?(resource['api_path']) + + msg = resource['info'] ? "#{type} - #{resource['info']}" : "#{type} at #{resource['api_path']}" + + puts "\nDeleting #{msg}..." + delete_response = delete(Runtime::API::Request.new(@api_client, resource['api_path']).url) + + if delete_response.code == 202 + print "\e[32m.\e[0m" + else + print "\e[31mF\e[0m" + failures << msg + end + end + end + + unless failures.empty? + puts "\nFailed to delete #{failures.length} resources:\n" + puts failures + end + end + + def resource_not_found?(api_path) + get_response = get Runtime::API::Request.new(@api_client, api_path).url + + get_response.code.eql? 404 + end + end + end +end diff --git a/qa/qa/tools/delete_test_ssh_keys.rb b/qa/qa/tools/delete_test_ssh_keys.rb index 58ab4865336..9e5728a5509 100644 --- a/qa/qa/tools/delete_test_ssh_keys.rb +++ b/qa/qa/tools/delete_test_ssh_keys.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_relative '../../qa' - # This script deletes all selected test ssh keys for a specific user # Keys can be selected by a string matching part of the key's title and by created date # - Specify `title_portion` to delete only keys that include the string provided diff --git a/qa/qa/tools/generate_perf_testdata.rb b/qa/qa/tools/generate_perf_testdata.rb index 8e5da94e7e6..0f06fd2fbc4 100644 --- a/qa/qa/tools/generate_perf_testdata.rb +++ b/qa/qa/tools/generate_perf_testdata.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'yaml' -require_relative '../../qa' + # This script generates testdata for Performance Testing. # Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS # This job creates a urls.txt which contains a hash of all the URLs needed for Performance Testing diff --git a/qa/qa/tools/initialize_gitlab_auth.rb b/qa/qa/tools/initialize_gitlab_auth.rb index 3ead8fc9bd4..86791f1f624 100644 --- a/qa/qa/tools/initialize_gitlab_auth.rb +++ b/qa/qa/tools/initialize_gitlab_auth.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_relative '../../qa' - module QA module Tools # Task to set default password from Runtime::Env.default_password if not set already diff --git a/qa/qa/tools/knapsack_report.rb b/qa/qa/tools/knapsack_report.rb index fb405e82e83..e50c4fe63d2 100644 --- a/qa/qa/tools/knapsack_report.rb +++ b/qa/qa/tools/knapsack_report.rb @@ -5,50 +5,86 @@ require "fog/google" module QA module Tools class KnapsackReport + extend SingleForwardable + PROJECT = "gitlab-qa-resources" BUCKET = "knapsack-reports" + FALLBACK_REPORT = "knapsack/master_report.json" - class << self - def download - new.download_report - end + def_delegators :new, :configure!, :move_regenerated_report, :download_report, :upload_report - def upload(glob) - new.upload_report(glob) - end - end + # Configure knapsack report + # + # * Setup variables + # * Fetch latest report + # + # @return [void] + def configure! + ENV["KNAPSACK_TEST_FILE_PATTERN"] ||= "qa/specs/features/**/*_spec.rb" + ENV["KNAPSACK_REPORT_PATH"] = report_path - def initialize - ENV["KNAPSACK_REPORT_PATH"] || raise("KNAPSACK_REPORT_PATH env var is required!") - ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise("QA_KNAPSACK_REPORT_GCS_CREDENTIALS env var is required!") + Knapsack.logger = QA::Runtime::Logger.logger + + download_report end # Download knapsack report from gcs bucket # # @return [void] def download_report - logger.info("Downloading latest knapsack report '#{report_file}'") + logger.debug("Downloading latest knapsack report for '#{report_name}' to '#{report_path}'") file = client.get_object(BUCKET, report_file) - - logger.info("Saving latest knapsack report to '#{report_path}'") File.write(report_path, file[:body]) + rescue StandardError => e + ENV["KNAPSACK_REPORT_PATH"] = FALLBACK_REPORT + logger.warn("Failed to fetch latest knapsack report: #{e}") + logger.warn("Falling back to '#{FALLBACK_REPORT}'") + end + + # Rename and move new regenerated report to a separate folder used to indicate report name + # + # @return [void] + def move_regenerated_report + return unless ENV["KNAPSACK_GENERATE_REPORT"] == "true" + + tmp_path = "tmp/knapsack/#{report_name}" + FileUtils.mkdir_p(tmp_path) + + # Use path from knapsack config in case of fallback to master_report.json + knapsack_report_path = Knapsack.report.report_path + logger.debug("Moving regenerated #{knapsack_report_path} to save as artifact") + FileUtils.cp(knapsack_report_path, "#{tmp_path}/#{ENV['CI_NODE_INDEX']}.json") end # Merge and upload knapsack report to gcs bucket # + # Fetches all files defined in glob and uses parent folder as report name + # # @param [String] glob # @return [void] def upload_report(glob) - reports = Dir[glob] - return logger.error("Pattern '#{glob}' did not match any files!") if reports.empty? + reports = Pathname.glob(glob).each_with_object(Hash.new { |hsh, key| hsh[key] = [] }) do |report, hash| + next unless report.extname == ".json" + + hash[report.parent.basename.to_s].push(report) + end + return logger.error("Glob '#{glob}' did not contain any valid report files!") if reports.empty? - report = reports - .map { |path| JSON.parse(File.read(path)) } - .reduce({}, :merge) - return logger.error("Knapsack generated empty report, skipping upload!") if report.empty? + reports.each do |name, jsons| + file = "#{name}.json" - logger.info("Uploading latest knapsack report '#{report_file}'") - client.put_object(BUCKET, report_file, JSON.pretty_generate(report)) + report = jsons + .map { |json| JSON.parse(File.read(json)) } + .reduce({}, :merge) + .sort_by { |k, v| v } # sort report by execution time + .to_h + next logger.warn("Knapsack generated empty report for '#{name}', skipping upload!") if report.empty? + + logger.info("Uploading latest knapsack report '#{file}'") + client.put_object(BUCKET, file, JSON.pretty_generate(report)) + rescue StandardError => e + logger.error("Failed to upload knapsack report for '#{name}'. Error: #{e}") + end end private @@ -64,24 +100,50 @@ module QA # # @return [Fog::Storage::GoogleJSON] def client - @client ||= Fog::Storage::Google.new( - google_project: PROJECT, - google_json_key_location: ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] - ) + @client ||= Fog::Storage::Google.new(google_project: PROJECT, **gcs_credentials) + end + + # Base path of knapsack report + # + # @return [String] + def report_base_path + @report_base_path ||= "knapsack" end # Knapsack report path # # @return [String] def report_path - @report_path ||= ENV["KNAPSACK_REPORT_PATH"] + @report_path ||= "#{report_base_path}/#{report_file}" end # Knapsack report name # # @return [String] def report_file - @report_name ||= report_path.split("/").last + @report_file ||= "#{report_name}.json" + end + + # Report name + # + # Infer report name from ci job name + # Remove characters incompatible with gcs bucket naming from job names like ee:instance-parallel + # + # @return [String] + def report_name + @report_name ||= ENV["CI_JOB_NAME"].split(" ").first.tr(":", "-") + end + + # GCS credentials json + # + # @return [Hash] + def gcs_credentials + json_key = ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise( + "QA_KNAPSACK_REPORT_GCS_CREDENTIALS env variable is required!" + ) + return { google_json_key_location: json_key } if File.exist?(json_key) + + { google_json_key_string: json_key } end end end diff --git a/qa/qa/tools/long_running_spec_reporter.rb b/qa/qa/tools/long_running_spec_reporter.rb new file mode 100644 index 00000000000..ce035248baa --- /dev/null +++ b/qa/qa/tools/long_running_spec_reporter.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require "fog/google" +require "slack-notifier" + +module QA + module Tools + class LongRunningSpecReporter + extend SingleForwardable + + SLACK_CHANNEL = "#quality-reports" + PROJECT = "gitlab-qa-resources" + BUCKET = "knapsack-reports" + REPORT_NAME = "ee-instance-parallel.json" + RUNTIME_THRESHOLD = 300 + + def_delegator :new, :execute + + # Find and report specs exceeding runtime threshold + # + # @return [void] + def execute + return puts("No long running specs detected, all good!") if long_running_specs.empty? + + specs = long_running_specs.map { |k, v| "#{k}: #{(v / 60).round(2)} minutes" }.join("\n") + average = mean_runtime < 60 ? "#{mean_runtime.round(0)} seconds" : "#{(mean_runtime / 60).round(2)} minutes" + msg = <<~MSG + Following spec files are exceeding #{RUNTIME_THRESHOLD / 60} minute runtime threshold! + Current average spec runtime: #{average}. + MSG + + puts("#{msg}\n#{specs}") + notifier.post(icon_emoji: ":time-out:", text: "#{msg}\n```#{specs}```") + end + + private + + # Average runtime of spec files + # + # @return [Number] + def mean_runtime + @mean_runtime ||= latest_report.values + .select { |v| v < RUNTIME_THRESHOLD } + .yield_self { |runtimes| runtimes.sum(0.0) / runtimes.length } + end + + # Spec files exceeding runtime threshold + # + # @return [Hash] + def long_running_specs + @long_running_specs ||= latest_report.select { |k, v| v > RUNTIME_THRESHOLD } + end + + # Latest knapsack report + # + # @return [Hash] + def latest_report + @latest_report ||= JSON.parse(client.get_object(BUCKET, REPORT_NAME)[:body]) + end + + # Slack notifier + # + # @return [Slack::Notifier] + def notifier + @notifier ||= Slack::Notifier.new( + slack_webhook_url, + channel: SLACK_CHANNEL, + username: "Spec Runtime Report" + ) + end + + # GCS client + # + # @return [Fog::Storage::GoogleJSON] + def client + @client ||= Fog::Storage::Google.new( + google_project: PROJECT, + **(File.exist?(gcs_json) ? { google_json_key_location: gcs_json } : { google_json_key_string: gcs_json }) + ) + end + + # Slack webhook url + # + # @return [String] + def slack_webhook_url + @slack_webhook_url ||= ENV["SLACK_WEBHOOK"] || raise("Missing SLACK_WEBHOOK env variable") + end + + # GCS credentials json + # + # @return [Hash] + def gcs_json + ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise("Missing QA_KNAPSACK_REPORT_GCS_CREDENTIALS env variable!") + end + end + end +end diff --git a/qa/qa/tools/reliable_report.rb b/qa/qa/tools/reliable_report.rb index 40a452be36e..b99b97c1ea6 100644 --- a/qa/qa/tools/reliable_report.rb +++ b/qa/qa/tools/reliable_report.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_relative "../../qa" - require "influxdb-client" require "terminal-table" require "slack-notifier" @@ -16,7 +14,7 @@ module QA PROJECT_ID = 278964 def initialize(range) - @range = range + @range = range.to_i @influxdb_bucket = "e2e-test-stats" @slack_channel = "#quality-reports" @influxdb_url = ENV["QA_INFLUXDB_URL"] || raise("Missing QA_INFLUXDB_URL env variable") @@ -34,9 +32,8 @@ module QA reporter.print_report reporter.report_in_issue_and_slack if report_in_issue_and_slack == "true" rescue StandardError => e - puts "Report creation failed! Error: '#{e}'".colorize(:red) - reporter.notify_failure(e) - exit(1) + reporter&.notify_failure(e) + raise(e) end # Print top stable specs @@ -58,7 +55,11 @@ module QA puts "Creating report".colorize(:green) response = post( "#{gitlab_api_url}/projects/#{PROJECT_ID}/issues", - { title: "Reliable spec report", description: report_issue_body, labels: "Quality,test" }, + { + title: "Reliable e2e test report", + description: report_issue_body, + labels: "Quality,test,type::maintenance,reliable test report" + }, headers: { "PRIVATE-TOKEN" => gitlab_access_token } ) web_url = parse_body(response)[:web_url] @@ -96,68 +97,79 @@ module QA # # @return [String] def report_issue_body + execution_interval = "(#{Date.today - range} - #{Date.today})" + issue = [] issue << "[[_TOC_]]" - issue << "# Candidates for promotion to reliable\n\n```\n#{stable_summary_table}\n```" - issue << results_markdown(stable_results_tables) + issue << "# Candidates for promotion to reliable #{execution_interval}" + issue << "Total amount: **#{stable_test_runs.sum { |_k, v| v.count }}**" + issue << stable_summary_table(markdown: true).to_s + issue << results_markdown(:stable) return issue.join("\n\n") if unstable_reliable_test_runs.empty? - issue << "# Reliable specs with failures\n\n```\n#{unstable_summary_table}\n```" - issue << results_markdown(unstable_reliable_results_tables) + issue << "# Reliable specs with failures #{execution_interval}" + issue << "Total amount: **#{unstable_reliable_test_runs.sum { |_k, v| v.count }}**" + issue << unstable_summary_table(markdown: true).to_s + issue << results_markdown(:unstable) issue.join("\n\n") end # Stable spec summary table # + # @param [Boolean] markdown # @return [Terminal::Table] - def stable_summary_table - @stable_summary_table ||= terminal_table( + def stable_summary_table(markdown: false) + terminal_table( rows: stable_test_runs.map { |stage, specs| [stage, specs.length] }, title: "Stable spec summary for past #{range} days".ljust(50), - headings: %w[STAGE COUNT] + headings: %w[STAGE COUNT], + markdown: markdown ) end # Unstable reliable summary table # + # @param [Boolean] markdown # @return [Terminal::Table] - def unstable_summary_table - @unstable_summary_table ||= terminal_table( + def unstable_summary_table(markdown: false) + terminal_table( rows: unstable_reliable_test_runs.map { |stage, specs| [stage, specs.length] }, title: "Unstable spec summary for past #{range} days".ljust(50), - headings: %w[STAGE COUNT] + headings: %w[STAGE COUNT], + markdown: markdown ) end # Result tables for stable specs # + # @param [Boolean] markdown # @return [Hash] - def stable_results_tables - @stable_results ||= results_tables(:stable) + def stable_results_tables(markdown: false) + results_tables(:stable, markdown: markdown) end # Result table for unstable specs # + # @param [Boolean] markdown # @return [Hash] - def unstable_reliable_results_tables - @unstable_results ||= results_tables(:unstable) + def unstable_reliable_results_tables(markdown: false) + results_tables(:unstable, markdown: markdown) end # Markdown formatted tables # - # @param [Hash] results + # @param [Symbol] type result type - :stable, :unstable # @return [String] - def results_markdown(results) - results.map do |stage, table| + def results_markdown(type) + runs = type == :stable ? stable_test_runs : unstable_reliable_test_runs + results_tables(type, markdown: true).map do |stage, table| <<~STAGE.strip - ## #{stage} + ## #{stage} (#{runs[stage].count}) <details> <summary>Executions table</summary> - ``` #{table} - ``` </details> STAGE @@ -167,15 +179,19 @@ module QA # Results table # # @param [Symbol] type result type - :stable, :unstable + # @param [Boolean] markdown # @return [Hash<Symbol, Terminal::Table>] - def results_tables(type) + def results_tables(type, markdown: false) (type == :stable ? stable_test_runs : unstable_reliable_test_runs).to_h do |stage, specs| headings = ["name", "runs", "failures", "failure rate"] [stage, terminal_table( - rows: specs.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] }, title: "Top #{type} specs in '#{stage}' stage for past #{range} days", - headings: headings.map(&:upcase) + headings: headings.map(&:upcase), + markdown: markdown, + rows: specs.map do |k, v| + [name_column(name: k, file: v[:file], markdown: markdown), *table_params(v.values)] + end )] end end @@ -214,13 +230,17 @@ module QA # Terminal table for result formatting # + # @param [Array] rows + # @param [Array] headings + # @param [String] title + # @param [Boolean] markdown # @return [Terminal::Table] - def terminal_table(rows:, headings:, title: nil) + def terminal_table(rows:, headings:, title:, markdown:) Terminal::Table.new( headings: headings, - style: { all_separators: true }, - title: title, - rows: rows + title: markdown ? nil : title, + rows: rows, + style: markdown ? { border: :markdown } : { all_separators: true } ) end @@ -232,17 +252,17 @@ module QA [*parameters[1..2], "#{parameters.last}%"] end - # Name column value + # Name column content # # @param [String] name # @param [String] file + # @param [Boolean] markdown # @return [String] - def name_column(name, file) - spec_name = name.length > 150 ? "#{name} ".scan(/.{1,150} /).map(&:strip).join("\n") : name - name_line = "name: '#{spec_name}'" - file_line = "file: '#{file}'" + def name_column(name:, file:, markdown: false) + return "**name**: #{name}<br>**file**: #{file}" if markdown - "#{name_line}\n#{file_line.ljust(160)}" + wrapped_name = name.length > 150 ? "#{name} ".scan(/.{1,150} /).map(&:strip).join("\n") : name + "name: '#{wrapped_name}'\nfile: #{file.ljust(160)}" end # Test executions grouped by name @@ -254,10 +274,16 @@ module QA all_runs = query_api.query(query: query(reliable)).values all_runs.each_with_object(Hash.new { |hsh, key| hsh[key] = {} }) do |table, result| - records = table.records - name = records.last.values["name"] - file = records.last.values["file_path"].split("/").last - stage = records.last.values["stage"] || "unknown" + records = table.records.sort_by { |record| record.values["_time"] } + # skip specs that executed less time than defined by range or stopped executing before report date + # offset 1 day due to how schedulers are configured and first run can be 1 day later + next if (Date.today - Date.parse(records.first.values["_time"])).to_i < (range - 1) + next if (Date.today - Date.parse(records.last.values["_time"])).to_i > 1 + + last_record = records.last.values + name = last_record["name"] + file = last_record["file_path"].split("/").last + stage = last_record["stage"] || "unknown" runs = records.count failed = records.count { |r| r.values["status"] == "failed" } diff --git a/qa/qa/tools/revoke_all_personal_access_tokens.rb b/qa/qa/tools/revoke_all_personal_access_tokens.rb index c0a1697fa16..b4fa02a36d4 100644 --- a/qa/qa/tools/revoke_all_personal_access_tokens.rb +++ b/qa/qa/tools/revoke_all_personal_access_tokens.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -require_relative '../../qa' require 'net/protocol' + # This script revokes all personal access tokens with the name of 'api-test-token' on the host specified by GITLAB_ADDRESS # Required environment variables: GITLAB_USERNAME, GITLAB_PASSWORD and GITLAB_ADDRESS # Run `rake revoke_personal_access_tokens` diff --git a/qa/qa/tools/test_resource_data_processor.rb b/qa/qa/tools/test_resource_data_processor.rb new file mode 100644 index 00000000000..78fb6ef6cd0 --- /dev/null +++ b/qa/qa/tools/test_resource_data_processor.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +# This script collects all resources created during each test execution +# Save the data and write it to a JSON file at the end of suite + +module QA + module Tools + class TestResourceDataProcessor + @resources ||= Hash.new { |hsh, key| hsh[key] = [] } + + class << self + # Ignoring rspec-mocks, sandbox, user and fork resources + # TODO: Will need to figure out which user resources can be collected, ignore for now + # + # Collecting resources created in E2E tests + # Data is a Hash of resources with keys as resource type (group, project, issue, etc.) + # Each type contains an array of resource object (hash) of the same type + # E.g: { "QA::Resource::Project": [ { info: 'foo', api_path: '/foo'}, {...} ] } + def collect(resource, info) + return if resource.api_response.nil? || + resource.is_a?(RSpec::Mocks::Double) || + resource.is_a?(Resource::Sandbox) || + resource.is_a?(Resource::User) || + resource.is_a?(Resource::Fork) + + api_path = if resource.respond_to?(:api_delete_path) + resource.api_delete_path.gsub('%2F', '/') + elsif resource.respond_to?(:api_get_path) + resource.api_get_path.gsub('%2F', '/') + else + 'Cannot find resource API path' + end + + type = resource.class.name + + @resources[type] << { info: info, api_path: api_path } + end + + # If JSON file exists and not empty, read and load file content + # Merge what is saved in @resources into the content from file + # Overwrite file content with the new data hash + # Otherwise create file and write data hash to file for the first time + def write_to_file + return if @resources.empty? + + file = Runtime::Env.test_resources_created_filepath + FileUtils.mkdir_p('tmp/') + FileUtils.touch(file) + data = nil + + if File.zero?(file) + data = @resources + else + data = JSON.parse(File.read(file)) + + @resources.each_pair do |key, val| + data[key].nil? ? data[key] = val : val.each { |item| data[key] << item } + end + end + + File.open(file, 'w') { |f| f.write(JSON.pretty_generate(data.each_value(&:uniq!))) } + end + end + end + end +end diff --git a/qa/spec/page/logging_spec.rb b/qa/spec/page/logging_spec.rb index 7c521f60b84..98326ecd343 100644 --- a/qa/spec/page/logging_spec.rb +++ b/qa/spec/page/logging_spec.rb @@ -14,6 +14,7 @@ RSpec.describe QA::Support::Page::Logging do allow(page).to receive(:find).and_return(page) allow(page).to receive(:current_url).and_return('http://current-url') allow(page).to receive(:has_css?).with(any_args).and_return(true) + allow(QA::Support::PageErrorChecker).to receive(:check_page_for_error_code).and_return(0) end subject do diff --git a/qa/spec/resource/base_spec.rb b/qa/spec/resource/base_spec.rb index 2a26a479436..2dd25f983bf 100644 --- a/qa/spec/resource/base_spec.rb +++ b/qa/spec/resource/base_spec.rb @@ -277,17 +277,30 @@ RSpec.describe QA::Resource::Base do describe '#visit!' do include_context 'with simple resource' + let(:wait_for_requests_class) { QA::Support::WaitForRequests } + before do allow(resource).to receive(:visit) end it 'calls #visit with the underlying #web_url' do allow(resource).to receive(:current_url).and_return(subject.current_url) + expect(wait_for_requests_class).to receive(:wait_for_requests).with({ skip_resp_code_check: false }).twice resource.web_url = subject.current_url resource.visit! expect(resource).to have_received(:visit).with(subject.current_url) end + + it 'calls #visit with the underlying #web_url with skip_resp_code_check specified as true' do + allow(resource).to receive(:current_url).and_return(subject.current_url) + expect(wait_for_requests_class).to receive(:wait_for_requests).with({ skip_resp_code_check: true }).twice + + resource.web_url = subject.current_url + resource.visit!(skip_resp_code_check: true) + + expect(resource).to have_received(:visit).with(subject.current_url) + end end end diff --git a/qa/spec/runtime/env_spec.rb b/qa/spec/runtime/env_spec.rb index 80d8a9a1892..0f752ad96b7 100644 --- a/qa/spec/runtime/env_spec.rb +++ b/qa/spec/runtime/env_spec.rb @@ -83,6 +83,46 @@ RSpec.describe QA::Runtime::Env do end end + describe '.running_on_dot_com?' do + using RSpec::Parameterized::TableSyntax + + where(:url, :result) do + 'https://www.gitlab.com' | true + 'https://staging.gitlab.com' | true + 'http://www.gitlab.com' | true + 'http://localhost:3000' | false + 'http://localhost' | false + 'http://gdk.test:3000' | false + end + + with_them do + before do + QA::Runtime::Scenario.define(:gitlab_address, url) + end + + it { expect(described_class.running_on_dot_com?).to eq result } + end + end + + describe '.running_on_dev?' do + using RSpec::Parameterized::TableSyntax + + where(:url, :result) do + 'https://www.gitlab.com' | false + 'http://localhost:3000' | true + 'http://localhost' | false + 'http://gdk.test:3000' | true + end + + with_them do + before do + QA::Runtime::Scenario.define(:gitlab_address, url) + end + + it { expect(described_class.running_on_dev?).to eq result } + end + end + describe '.personal_access_token' do around do |example| described_class.instance_variable_set(:@personal_access_token, nil) @@ -173,31 +213,23 @@ RSpec.describe QA::Runtime::Env do stub_env('CI_NODE_TOTAL', '2') end - it 'returns true if KNAPSACK_GENERATE_REPORT is defined' do - stub_env('KNAPSACK_GENERATE_REPORT', 'true') - + it 'returns true if running in parallel CI run' do expect(described_class.knapsack?).to be_truthy end - it 'returns true if KNAPSACK_REPORT_PATH is defined' do - stub_env('KNAPSACK_REPORT_PATH', '/a/path') - - expect(described_class.knapsack?).to be_truthy + it 'returns false if knapsack disabled' do + stub_env('NO_KNAPSACK', 'true') + expect(described_class.knapsack?).to be_falsey end - it 'returns true if KNAPSACK_TEST_FILE_PATTERN is defined' do - stub_env('KNAPSACK_TEST_FILE_PATTERN', '/a/**/pattern') - - expect(described_class.knapsack?).to be_truthy - end + it 'returns false if not running in a parallel job' do + stub_env('CI_NODE_TOTAL', '1') - it 'returns false if neither KNAPSACK_GENERATE_REPORT nor KNAPSACK_REPORT_PATH nor KNAPSACK_TEST_FILE_PATTERN are defined' do expect(described_class.knapsack?).to be_falsey end - it 'returns false if not running in parallel job' do - stub_env('CI_NODE_TOTAL', '1') - stub_env('KNAPSACK_GENERATE_REPORT', 'true') + it 'returns false if not running in ci' do + stub_env('CI_NODE_TOTAL', nil) expect(described_class.knapsack?).to be_falsey end @@ -328,4 +360,36 @@ RSpec.describe QA::Runtime::Env do end end end + + describe '.test_resources_created_filepath' do + context 'when not in CI' do + before do + allow(described_class).to receive(:running_in_ci?).and_return(false) + end + + it 'returns default path if QA_TEST_RESOURCES_CREATED_FILEPATH is not defined' do + stub_env('QA_TEST_RESOURCES_CREATED_FILEPATH', nil) + + expect(described_class.test_resources_created_filepath).to include('tmp/test-resources.json') + end + + it 'returns path if QA_TEST_RESOURCES_CREATED_FILEPATH is defined' do + stub_env('QA_TEST_RESOURCES_CREATED_FILEPATH', 'path/to_file') + + expect(described_class.test_resources_created_filepath).to eq('path/to_file') + end + end + + context 'when in CI' do + before do + allow(described_class).to receive(:running_in_ci?).and_return(true) + allow(SecureRandom).to receive(:hex).with(3).and_return('abc123') + stub_env('QA_TEST_RESOURCES_CREATED_FILEPATH', nil) + end + + it 'returns path with random hex in file name' do + expect(described_class.test_resources_created_filepath).to include('tmp/test-resources-abc123.json') + end + end + end end diff --git a/qa/spec/scenario/test/integration/github_spec.rb b/qa/spec/scenario/test/integration/github_spec.rb index b68b06a7b9f..4ad42b4f5cc 100644 --- a/qa/spec/scenario/test/integration/github_spec.rb +++ b/qa/spec/scenario/test/integration/github_spec.rb @@ -2,7 +2,7 @@ RSpec.describe QA::Scenario::Test::Integration::Github do describe '#perform' do - let(:env) { spy('Runtime::Env') } + let(:env) { spy('Runtime::Env', knapsack?: false, dry_run: false) } before do stub_const('QA::Runtime::Env', env) diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb index 4372d9d2728..47791f76970 100644 --- a/qa/spec/spec_helper.rb +++ b/qa/spec/spec_helper.rb @@ -10,9 +10,9 @@ require 'active_support/core_ext/object/blank' require_relative 'qa_deprecation_toolkit_env' QaDeprecationToolkitEnv.configure! -Knapsack::Adapters::RSpecAdapter.bind if ENV['CI'] && QA::Runtime::Env.knapsack? && !ENV['NO_KNAPSACK'] +Knapsack::Adapters::RSpecAdapter.bind if QA::Runtime::Env.knapsack? -QA::Runtime::Browser.configure! +QA::Runtime::Browser.configure! unless QA::Runtime::Env.dry_run QA::Runtime::AllureReport.configure! QA::Runtime::Scenario.from_env(QA::Runtime::Env.runtime_scenario_attributes) @@ -66,9 +66,14 @@ RSpec.configure do |config| config.after(:suite) do |suite| # If any tests failed, leave the resources behind to help troubleshoot - next if suite.reporter.failed_examples.present? + QA::Resource::ReusableProject.remove_all_via_api! unless suite.reporter.failed_examples.present? - QA::Resource::ReusableProject.remove_all_via_api! + # Write all test created resources to JSON file + QA::Tools::TestResourceDataProcessor.write_to_file + end + + config.append_after(:suite) do + QA::Tools::KnapsackReport.move_regenerated_report if QA::Runtime::Env.knapsack? end config.expect_with :rspec do |expectations| @@ -95,8 +100,14 @@ RSpec.configure do |config| if ENV['CI'] && !QA::Runtime::Env.disable_rspec_retry? non_quarantine_retries = QA::Runtime::Env.ci_project_name =~ /staging|canary|production/ ? 3 : 2 config.around do |example| - retry_times = example.metadata.key?(:quarantine) ? 1 : non_quarantine_retries - example.run_with_retry retry: retry_times + quarantine = example.metadata[:quarantine] + different_quarantine_context = QA::Specs::Helpers::Quarantine.quarantined_different_context?(quarantine) + focused_quarantine = QA::Specs::Helpers::Quarantine.filters.key?(:quarantine) + + # Do not disable retry when spec is quarantined but on different environment + next example.run_with_retry(retry: non_quarantine_retries) if different_quarantine_context && !focused_quarantine + + example.run_with_retry(retry: quarantine ? 1 : non_quarantine_retries) end end end diff --git a/qa/spec/specs/runner_spec.rb b/qa/spec/specs/runner_spec.rb index 021ce9091e0..5cc9ff403cd 100644 --- a/qa/spec/specs/runner_spec.rb +++ b/qa/spec/specs/runner_spec.rb @@ -28,6 +28,26 @@ RSpec.describe QA::Specs::Runner do end end + context 'when count_examples_only is set as an option' do + let(:out) { StringIO.new } + + before do + QA::Runtime::Scenario.define(:count_examples_only, true) + out.string = '22 examples,' + allow(StringIO).to receive(:new).and_return(out) + end + + it 'sets the `--dry-run` flag' do + expect_rspec_runner_arguments(['--dry-run', '--tag', '~orchestrated', '--tag', '~transient', '--tag', '~geo', *described_class::DEFAULT_TEST_PATH_ARGS], [$stderr, anything]) + + subject.perform + end + + after do + QA::Runtime::Scenario.attributes.delete(:count_examples_only) + end + end + context 'when tags are set' do subject { described_class.new.tap { |runner| runner.tags = %i[orchestrated github] } } @@ -158,10 +178,10 @@ RSpec.describe QA::Specs::Runner do end end - def expect_rspec_runner_arguments(arguments) + def expect_rspec_runner_arguments(arguments, std_arguments = described_class::DEFAULT_STD_ARGS) expect(RSpec::Core::Runner).to receive(:run) - .with(arguments, $stderr, $stdout) - .and_return(0) + .with(arguments, *std_arguments) + .and_return(0) end end end diff --git a/qa/spec/support/formatters/test_stats_formatter_spec.rb b/qa/spec/support/formatters/test_stats_formatter_spec.rb index 71ab9c1d541..2bfd7863653 100644 --- a/qa/spec/support/formatters/test_stats_formatter_spec.rb +++ b/qa/spec/support/formatters/test_stats_formatter_spec.rb @@ -56,13 +56,14 @@ describe QA::Support::Formatters::TestStatsFormatter do retry_attempts: 0, job_url: ci_job_url, pipeline_url: ci_pipeline_url, - pipeline_id: ci_pipeline_id + pipeline_id: ci_pipeline_id, + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234' } } end def run_spec(&spec) - spec ||= -> { it('spec') {} } + spec ||= -> { it('spec', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {} } describe_successfully('stats export', &spec).tap do |example_group| example_group.examples.each { |ex| ex.metadata[:file_path] = file_path } @@ -131,7 +132,7 @@ describe QA::Support::Formatters::TestStatsFormatter do it 'exports data to influxdb with correct reliable tag' do run_spec do - it('spec', :reliable) {} + it('spec', :reliable, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {} end expect(influx_write_api).to have_received(:write).with(data: [data]) @@ -143,7 +144,7 @@ describe QA::Support::Formatters::TestStatsFormatter do it 'exports data to influxdb with correct quarantine tag' do run_spec do - it('spec', :quarantine) {} + it('spec', :quarantine, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {} end expect(influx_write_api).to have_received(:write).with(data: [data]) diff --git a/qa/spec/support/page_error_checker_spec.rb b/qa/spec/support/page_error_checker_spec.rb new file mode 100644 index 00000000000..764b6110e08 --- /dev/null +++ b/qa/spec/support/page_error_checker_spec.rb @@ -0,0 +1,217 @@ +# frozen_string_literal: true + +RSpec.describe QA::Support::PageErrorChecker do + let(:test_path) { '/test/path' } + + let(:page) { double(Capybara.page) } + + describe '.report!' do + context 'reports errors' do + let(:expected_chrome_error) do + "chrome errors\n\n"\ + "Path: #{test_path}" + end + + let(:expected_basic_error) do + "foo status\n\n"\ + "Path: #{test_path}" + end + + it 'reports error message on chrome browser' do + allow(QA::Support::PageErrorChecker).to receive(:return_chrome_errors).and_return('chrome errors') + allow(page).to receive(:current_path).and_return(test_path) + allow(QA::Runtime::Env).to receive(:browser).and_return(:chrome) + + expect { QA::Support::PageErrorChecker.report!(page, 500) }.to raise_error(RuntimeError, expected_chrome_error) + end + + it 'reports basic message on non-chrome browser' do + allow(QA::Support::PageErrorChecker).to receive(:status_code_report).and_return('foo status') + allow(page).to receive(:current_path).and_return(test_path) + allow(QA::Runtime::Env).to receive(:browser).and_return(:firefox) + + expect { QA::Support::PageErrorChecker.report!(page, 500) }.to raise_error(RuntimeError, expected_basic_error) + end + end + end + + describe '.return_chrome_errors' do + context 'returns error message' do + before do + single_log = Class.new do + def level + 'SEVERE' + end + end + stub_const('SingleLog', single_log) + one_error_mocked_logs = Class.new do + def self.select + [SingleLog] + end + end + stub_const('OneErrorMockedLogs', one_error_mocked_logs) + three_errors_mocked_logs = Class.new do + def self.select + [SingleLog, SingleLog, SingleLog] + end + end + stub_const('ThreeErrorsMockedLogs', three_errors_mocked_logs) + no_error_mocked_logs = Class.new do + def self.select + [] + end + end + stub_const('NoErrorMockedLogs', no_error_mocked_logs) + end + + let(:expected_single_error) do + "There was 1 SEVERE level error:\n\n"\ + "bar foo" + end + + let(:expected_multiple_error) do + "There were 3 SEVERE level errors:\n\n"\ + "bar foo\n"\ + "foo\n"\ + "bar" + end + + it 'returns status code report on no severe errors found' do + allow(QA::Support::PageErrorChecker).to receive(:logs).with(page).and_return(NoErrorMockedLogs) + allow(QA::Support::PageErrorChecker).to receive(:status_code_report).with('123').and_return('Test Status Code return 123') + + expect(QA::Support::PageErrorChecker.return_chrome_errors(page, '123')).to eq('Test Status Code return 123') + end + + it 'returns report on 1 severe error found' do + allow(QA::Support::PageErrorChecker).to receive(:error_report_for).with([SingleLog]).and_return('bar foo') + allow(QA::Support::PageErrorChecker).to receive(:logs).with(page).and_return(OneErrorMockedLogs) + allow(page).to receive(:current_path).and_return(test_path) + + expect(QA::Support::PageErrorChecker.return_chrome_errors(page, '123')).to eq(expected_single_error) + end + + it 'returns report on multiple severe errors found' do + allow(QA::Support::PageErrorChecker).to receive(:error_report_for) + .with([SingleLog, SingleLog, SingleLog]).and_return("bar foo\nfoo\nbar") + allow(QA::Support::PageErrorChecker).to receive(:logs).with(page).and_return(ThreeErrorsMockedLogs) + allow(page).to receive(:current_path).and_return(test_path) + + expect(QA::Support::PageErrorChecker.return_chrome_errors(page, '123')).to eq(expected_multiple_error) + end + end + end + + describe '.check_page_for_error_code' do + require 'nokogiri' + before do + nokogiri_parse = Class.new do + def self.parse(str) + Nokogiri::HTML.parse(str) + end + end + stub_const('NokogiriParse', nokogiri_parse) + end + let(:error_404_str) do + "<div class=\"error\">"\ + "<img src=\"404.png\" alt=\"404\" />"\ + "</div>" + end + + let(:error_500_str) { "<h1> 500 </h1>"} + let(:backtrace_str) {"<body><section class=\"backtrace\">foo</section></body>"} + let(:no_error_str) {"<body>no 404 or 500 or backtrace</body>"} + + it 'calls report with 404 if 404 found' do + allow(page).to receive(:html).and_return(error_404_str) + allow(Nokogiri::HTML).to receive(:parse).with(error_404_str).and_return(NokogiriParse.parse(error_404_str)) + + expect(QA::Support::PageErrorChecker).to receive(:report!).with(page, 404) + QA::Support::PageErrorChecker.check_page_for_error_code(page) + end + it 'calls report with 500 if 500 found' do + allow(page).to receive(:html).and_return(error_500_str) + allow(Nokogiri::HTML).to receive(:parse).with(error_500_str).and_return(NokogiriParse.parse(error_500_str)) + + expect(QA::Support::PageErrorChecker).to receive(:report!).with(page, 500) + QA::Support::PageErrorChecker.check_page_for_error_code(page) + end + it 'calls report with 500 if GDK backtrace found' do + allow(page).to receive(:html).and_return(backtrace_str) + allow(Nokogiri::HTML).to receive(:parse).with(backtrace_str).and_return(NokogiriParse.parse(backtrace_str)) + + expect(QA::Support::PageErrorChecker).to receive(:report!).with(page, 500) + QA::Support::PageErrorChecker.check_page_for_error_code(page) + end + it 'does not call report if no 404, 500 or backtrace found' do + allow(page).to receive(:html).and_return(no_error_str) + allow(Nokogiri::HTML).to receive(:parse).with(no_error_str).and_return(NokogiriParse.parse(no_error_str)) + + expect(QA::Support::PageErrorChecker).not_to receive(:report!) + QA::Support::PageErrorChecker.check_page_for_error_code(page) + end + end + + describe '.error_report_for' do + before do + logs_class_one = Class.new do + def self.message + 'foo\\n' + end + end + stub_const('LogOne', logs_class_one) + logs_class_two = Class.new do + def self.message + 'bar' + end + end + stub_const('LogTwo', logs_class_two) + end + + it 'returns error report array of log messages' do + expect(QA::Support::PageErrorChecker.error_report_for([LogOne, LogTwo])) + .to eq(%W(foo\n bar)) + end + end + + describe '.logs' do + before do + logs_class = Class.new do + def self.get(level) + "logs at #{level} level" + end + end + stub_const('Logs', logs_class) + manage_class = Class.new do + def self.logs + Logs + end + end + stub_const('Manage', manage_class) + browser_class = Class.new do + def self.manage + Manage + end + end + stub_const('Browser', browser_class) + driver_class = Class.new do + def self.browser + Browser + end + end + stub_const('Driver', driver_class) + end + + it 'gets driver browser logs' do + allow(page).to receive(:driver).and_return(Driver) + + expect(QA::Support::PageErrorChecker.logs(page)).to eq('logs at browser level') + end + end + + describe '.status_code_report' do + it 'returns a string message containing the status code' do + expect(QA::Support::PageErrorChecker.status_code_report(1234)).to eq('Status code 1234 found') + end + end +end diff --git a/qa/spec/support/wait_for_requests_spec.rb b/qa/spec/support/wait_for_requests_spec.rb index 47c35addd9f..2492820b67f 100644 --- a/qa/spec/support/wait_for_requests_spec.rb +++ b/qa/spec/support/wait_for_requests_spec.rb @@ -22,5 +22,21 @@ RSpec.describe QA::Support::WaitForRequests do subject.wait_for_requests(skip_finished_loading_check: true) end end + + context 'when skip_resp_code_check is defaulted to false' do + it 'call report' do + allow(QA::Support::PageErrorChecker).to receive(:check_page_for_error_code).with(Capybara.page) + + subject.wait_for_requests + end + end + + context 'when skip_resp_code_check is true' do + it 'does not parse for an error code' do + expect(QA::Support::PageErrorChecker).not_to receive(:check_page_for_error_code) + + subject.wait_for_requests(skip_resp_code_check: true) + end + end end end diff --git a/qa/spec/tools/long_running_spec_reporter_spec.rb b/qa/spec/tools/long_running_spec_reporter_spec.rb new file mode 100644 index 00000000000..1bf520c53af --- /dev/null +++ b/qa/spec/tools/long_running_spec_reporter_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +RSpec.describe QA::Tools::LongRunningSpecReporter do + include QA::Support::Helpers::StubEnv + + subject(:reporter) { described_class.execute } + + let(:gcs_client) { double("Fog::Storage::GoogleJSON", get_object: report) } + let(:slack_notifier) { double("Slack::Notifier", post: nil) } + + before do + stub_env("SLACK_WEBHOOK", "slack_url") + stub_env("QA_KNAPSACK_REPORT_GCS_CREDENTIALS", "gcs_json") + + allow(Fog::Storage::Google).to receive(:new) + .with(google_project: "gitlab-qa-resources", google_json_key_string: "gcs_json") + .and_return(gcs_client) + allow(Slack::Notifier).to receive(:new) + .with("slack_url", channel: "#quality-reports", username: "Spec Runtime Report") + .and_return(slack_notifier) + end + + context "without specs exceeding runtime" do + let(:report) do + { + body: <<~JSON + { + "spec.rb": 5, + "spec_2.rb": 10 + } + JSON + } + end + + it "returns all good message" do + expect { reporter }.to output("No long running specs detected, all good!\n").to_stdout + end + end + + context "with specs exceeding runtime" do + let(:report) do + { + body: <<~JSON + { + "spec.rb": 5.0, + "spec_2.rb": 320.0 + } + JSON + } + end + + let(:spec) { "spec_2.rb: 5.33 minutes" } + + let(:message) do + <<~MSG + Following spec files are exceeding 5 minute runtime threshold! + Current average spec runtime: 5 seconds. + MSG + end + + it "notifies on long running specs" do + expect { reporter }.to output("#{message}\n#{spec}\n").to_stdout + expect(slack_notifier).to have_received(:post).with( + icon_emoji: ":time-out:", + text: "#{message}\n```#{spec}```" + ) + end + end +end diff --git a/qa/spec/tools/reliable_report_spec.rb b/qa/spec/tools/reliable_report_spec.rb index a048aa2e6ea..1ff62df34e0 100644 --- a/qa/spec/tools/reliable_report_spec.rb +++ b/qa/spec/tools/reliable_report_spec.rb @@ -13,30 +13,43 @@ describe QA::Tools::ReliableReport do let(:slack_channel) { "#quality-reports" } let(:range) { 14 } let(:issue_url) { "https://gitlab.com/issue/1" } + let(:time) { "2021-12-07T04:05:25.000000000+00:00" } let(:runs) do - values = { "name" => "stable spec", "status" => "passed", "file_path" => "some/spec.rb", "stage" => "manage" } + values = { + "name" => "stable spec", + "status" => "passed", + "file_path" => "some/spec.rb", + "stage" => "manage", + "_time" => time + } { 0 => instance_double( "InfluxDB2::FluxTable", records: [ instance_double("InfluxDB2::FluxRecord", values: values), instance_double("InfluxDB2::FluxRecord", values: values), - instance_double("InfluxDB2::FluxRecord", values: values) + instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s })) ] ) } end let(:reliable_runs) do - values = { "name" => "unstable spec", "status" => "failed", "file_path" => "some/spec.rb", "stage" => "create" } + values = { + "name" => "unstable spec", + "status" => "failed", + "file_path" => "some/spec.rb", + "stage" => "create", + "_time" => time + } { 0 => instance_double( "InfluxDB2::FluxTable", records: [ instance_double("InfluxDB2::FluxRecord", values: { **values, "status" => "passed" }), instance_double("InfluxDB2::FluxRecord", values: values), - instance_double("InfluxDB2::FluxRecord", values: values) + instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s })) ] ) } @@ -67,41 +80,34 @@ describe QA::Tools::ReliableReport do def markdown_section(summary, result, stage, type) <<~SECTION.strip - ``` - #{summary_table(summary, type)} - ``` + #{summary_table(summary, type, true)} - ## #{stage} + ## #{stage} (1) <details> <summary>Executions table</summary> - ``` - #{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days")} - ``` + #{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days", true)} </details> SECTION end - def summary_table(summary, type) - table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50)) + def summary_table(summary, type, markdown = false) + table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50), markdown) end - def table(rows, headings, title) + def table(rows, headings, title, markdown = false) Terminal::Table.new( headings: headings, - style: { all_separators: true }, - title: title, - rows: rows + title: markdown ? nil : title, + rows: rows, + style: markdown ? { border: :markdown } : { all_separators: true } ) end def name_column(spec_name) - name = "name: '#{spec_name}'" - file = "file: 'spec.rb'".ljust(160) - - "#{name}\n#{file}" + "**name**: #{spec_name}<br>**file**: spec.rb" end before do @@ -136,11 +142,15 @@ describe QA::Tools::ReliableReport do <<~TXT.strip [[_TOC_]] - # Candidates for promotion to reliable + # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today}) + + Total amount: **1** #{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')} - # Reliable specs with failures + # Reliable specs with failures (#{Date.today - range} - #{Date.today}) + + Total amount: **1** #{markdown_section([['create', 1]], [[name_column('unstable spec'), 3, 2, '66.67%']], 'create', 'unstable')} TXT @@ -155,9 +165,9 @@ describe QA::Tools::ReliableReport do verify_ssl: false, headers: { "PRIVATE-TOKEN" => "gitlab_token" }, payload: { - title: "Reliable spec report", + title: "Reliable e2e test report", description: issue_body, - labels: "Quality,test" + labels: "Quality,test,type::maintenance,reliable test report" } ) expect(slack_notifier).to have_received(:post).with( @@ -180,7 +190,7 @@ describe QA::Tools::ReliableReport do end it "notifies failure", :aggregate_failures do - expect { expect { run }.to raise_error(SystemExit) }.to output.to_stdout + expect { expect { run }.to raise_error("Connection error!") }.to output.to_stdout expect(slack_notifier).to have_received(:post).with( icon_emoji: ":sadpanda:", diff --git a/qa/spec/tools/test_resources_data_processor_spec.rb b/qa/spec/tools/test_resources_data_processor_spec.rb new file mode 100644 index 00000000000..6a8c0fd06a4 --- /dev/null +++ b/qa/spec/tools/test_resources_data_processor_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +RSpec.describe QA::Tools::TestResourceDataProcessor do + let(:info) { 'information' } + let(:api_path) { '/foo' } + let(:result) { [{ info: info, api_path: api_path }] } + + describe '.collect' do + context 'when resource is not restricted' do + let(:resource) { instance_double(QA::Resource::Project, api_delete_path: '/foo', api_response: 'foo') } + + it 'collects resource' do + expect(described_class.collect(resource, info)).to eq(result) + end + end + + context 'when resource api response is nil' do + let(:resource) { double(QA::Resource::Project, api_delete_path: '/foo', api_response: nil) } + + it 'does not collect resource' do + expect(described_class.collect(resource, info)).to eq(nil) + end + end + + context 'when resource is restricted' do + let(:resource) { double(QA::Resource::Sandbox, api_delete_path: '/foo', api_response: 'foo') } + + it 'does not collect resource' do + expect(described_class.collect(resource, info)).to eq(nil) + end + end + end +end diff --git a/qa/tasks/knapsack.rake b/qa/tasks/knapsack.rake index ea15793a457..ce4a1679bd1 100644 --- a/qa/tasks/knapsack.rake +++ b/qa/tasks/knapsack.rake @@ -1,8 +1,6 @@ # frozen_string_literal: true # rubocop:disable Rails/RakeEnvironment -require_relative "../qa/tools/knapsack_report" - namespace :knapsack do desc "Download latest knapsack report" task :download do @@ -10,8 +8,13 @@ namespace :knapsack do end desc "Merge and upload knapsack report" - task :upload, [:glob_pattern] do |_task, args| - QA::Tools::KnapsackReport.upload(args[:glob_pattern]) + task :upload, [:glob] do |_task, args| + QA::Tools::KnapsackReport.upload_report(args[:glob]) + end + + desc "Report long running spec files" + task :notify_long_running_specs do + QA::Tools::LongRunningSpecReporter.execute end end # rubocop:enable Rails/RakeEnvironment diff --git a/qa/tasks/reliable_report.rake b/qa/tasks/reliable_report.rake index 4ec86779704..b4dcc2ebc01 100644 --- a/qa/tasks/reliable_report.rake +++ b/qa/tasks/reliable_report.rake @@ -1,8 +1,6 @@ # frozen_string_literal: true # rubocop:disable Rails/RakeEnvironment -require_relative "../qa/tools/reliable_report" - desc "Fetch reliable and unreliable spec data and create report" task :reliable_spec_report, [:range, :report_in_issue_and_slack] do |_task, args| QA::Tools::ReliableReport.run(**args) |