diff options
Diffstat (limited to 'qa/qa')
87 files changed, 1864 insertions, 813 deletions
diff --git a/qa/qa/fixtures/metrics_dashboards/templating.yml b/qa/qa/fixtures/metrics_dashboards/templating.yml index e06e7cc1247..847eba59bd2 100644 --- a/qa/qa/fixtures/metrics_dashboards/templating.yml +++ b/qa/qa/fixtures/metrics_dashboards/templating.yml @@ -40,4 +40,4 @@ panel_groups: - id: pod_memory_working_set1 query_range: 'container_memory_working_set_bytes{pod_name=~"{{pod_name2}}"}/1024/1024' unit: "MiB" - label: pod_name + label: pod_name
\ No newline at end of file diff --git a/qa/qa/flow/login.rb b/qa/qa/flow/login.rb index 5f7e0227ac5..b60f74fe9bf 100644 --- a/qa/qa/flow/login.rb +++ b/qa/qa/flow/login.rb @@ -6,8 +6,6 @@ module QA module_function def while_signed_in(as: nil, address: :gitlab, admin: false) - Page::Main::Menu.perform(&:sign_out_if_signed_in) - sign_in(as: as, address: address, admin: admin) result = yield @@ -23,9 +21,10 @@ module QA end def sign_in(as: nil, address: :gitlab, skip_page_validation: false, admin: false) + Page::Main::Login.perform { |p| p.redirect_to_login_page(address) } + unless Page::Main::Login.perform(&:on_login_page?) Page::Main::Menu.perform(&:sign_out) if Page::Main::Menu.perform(&:signed_in?) - Runtime::Browser.visit(address, Page::Main::Login) end Page::Main::Login.perform do |login| diff --git a/qa/qa/flow/sign_up.rb b/qa/qa/flow/sign_up.rb index a2a62371092..ec7886ef969 100644 --- a/qa/qa/flow/sign_up.rb +++ b/qa/qa/flow/sign_up.rb @@ -26,9 +26,12 @@ module QA sign_up.click_new_user_register_button end - Page::Registration::Welcome.perform(&:click_get_started_button_if_available) + Flow::UserOnboarding.onboard_user success = if user.expect_fabrication_success + # In development env and .com the user is asked to create a group and a project which can be skipped for + # the purpose of signing up + Runtime::Browser.visit(:gitlab, Page::Dashboard::Welcome) Page::Main::Menu.perform(&:has_personal_area?) else Page::Main::Menu.perform(&:not_signed_in?) diff --git a/qa/qa/flow/user_onboarding.rb b/qa/qa/flow/user_onboarding.rb new file mode 100644 index 00000000000..066e1878869 --- /dev/null +++ b/qa/qa/flow/user_onboarding.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module QA + module Flow + module UserOnboarding + module_function + + def onboard_user + Page::Registration::Welcome.perform do |welcome_page| + if welcome_page.has_get_started_button? + welcome_page.select_role('Other') + welcome_page.choose_setup_for_company_if_available + welcome_page.click_get_started_button + end + end + end + end + end +end diff --git a/qa/qa/page/base.rb b/qa/qa/page/base.rb index 69f58dcb8a5..526dd25ccc9 100644 --- a/qa/qa/page/base.rb +++ b/qa/qa/page/base.rb @@ -386,6 +386,10 @@ module QA end end + def current_host + URI(page.current_url).host + end + def self.path raise NotImplementedError end diff --git a/qa/qa/page/component/confirm_modal.rb b/qa/qa/page/component/confirm_modal.rb index 25eea8e0d93..a90be76c879 100644 --- a/qa/qa/page/component/confirm_modal.rb +++ b/qa/qa/page/component/confirm_modal.rb @@ -8,12 +8,6 @@ module QA def self.included(base) super - - base.view 'app/views/shared/_confirm_modal.html.haml' do - element :confirm_modal - element :confirm_input - element :confirm_button - end end def fill_confirmation_text(text) diff --git a/qa/qa/page/component/invite_members_modal.rb b/qa/qa/page/component/invite_members_modal.rb index fecd61fb410..ca6862ccb02 100644 --- a/qa/qa/page/component/invite_members_modal.rb +++ b/qa/qa/page/component/invite_members_modal.rb @@ -47,37 +47,43 @@ module QA fill_element :members_token_select_input, username Support::WaitForRequests.wait_for_requests click_button username - - # Guest option is selected by default, skipping these steps if desired option is 'Guest' - unless access_level == 'Guest' - click_element :access_level_dropdown - click_button access_level - end - - click_element :invite_button + set_access_level(access_level) end - Support::WaitForRequests.wait_for_requests - - page.refresh + send_invite end - def invite_group(group_name, group_access = Resource::Members::AccessLevel::GUEST) + def invite_group(group_name, access_level = 'Guest') open_invite_group_modal - fill_element :access_level_dropdown, with: group_access + within_element(:invite_members_modal_content) do + click_button 'Select a group' - click_button 'Select a group' - fill_element :group_select_dropdown_search_field, group_name + # Helps stabilize race condition with concurrent group API calls while searching + # TODO: Replace with `fill_element :group_select_dropdown_search_field, group_name` when this bug is resolved: https://gitlab.com/gitlab-org/gitlab/-/issues/349379 + send_keys_to_element(:group_select_dropdown_search_field, group_name) - Support::WaitForRequests.wait_for_requests + Support::WaitForRequests.wait_for_requests + click_button group_name + set_access_level(access_level) + end - click_button group_name + send_invite + end - click_element :invite_button + private - Support::WaitForRequests.wait_for_requests + def set_access_level(access_level) + # Guest option is selected by default, skipping these steps if desired option is 'Guest' + unless access_level == 'Guest' + click_element :access_level_dropdown + click_button access_level + end + end + def send_invite + click_element :invite_button + Support::WaitForRequests.wait_for_requests page.refresh end end diff --git a/qa/qa/page/dashboard/welcome.rb b/qa/qa/page/dashboard/welcome.rb index b54205780d9..6f645e168c7 100644 --- a/qa/qa/page/dashboard/welcome.rb +++ b/qa/qa/page/dashboard/welcome.rb @@ -11,6 +11,10 @@ module QA def has_welcome_title?(text) has_element?(:welcome_title_content, text: text) end + + def self.path + '/' + end end end end diff --git a/qa/qa/page/file/show.rb b/qa/qa/page/file/show.rb index 730c5a88515..e54c3e0cd07 100644 --- a/qa/qa/page/file/show.rb +++ b/qa/qa/page/file/show.rb @@ -23,8 +23,26 @@ module QA element :delete_file_button, "button_tag 'Delete file'" # rubocop:disable QA/ElementWithPattern end + view 'app/assets/javascripts/vue_shared/components/web_ide_link.vue' do + element :edit_button + end + + view 'app/assets/javascripts/vue_shared/components/actions_button.vue' do + element :action_dropdown + element :edit_menu_item, ':data-qa-selector="`${action.key}_menu_item`"' # rubocop:disable QA/ElementWithPattern + end + def click_edit - click_on 'Edit' + # TODO: remove this condition and else part once ff :consolidated_edit_button is enabled by default + if has_element?(:action_dropdown) + within_element(:action_dropdown) do + click_button(class: 'dropdown-toggle-split') + click_element(:edit_menu_item) + click_element(:edit_button) + end + else + click_on 'Edit' + end end def click_delete diff --git a/qa/qa/page/main/login.rb b/qa/qa/page/main/login.rb index 5cba9d4bce4..f004107d7bd 100644 --- a/qa/qa/page/main/login.rb +++ b/qa/qa/page/main/login.rb @@ -156,6 +156,11 @@ module QA sign_in_using_credentials(user: user) end + def redirect_to_login_page(address) + desired_host = URI(Runtime::Scenario.send("#{address}_address")).host + Runtime::Browser.visit(address, Page::Main::Login) if desired_host != current_host + end + private def sign_in_using_gitlab_credentials(user:, skip_page_validation: false) diff --git a/qa/qa/page/project/branches/show.rb b/qa/qa/page/project/branches/show.rb index afec0e27a0b..a19fcf8ec6e 100644 --- a/qa/qa/page/project/branches/show.rb +++ b/qa/qa/page/project/branches/show.rb @@ -14,7 +14,6 @@ module QA end view 'app/views/projects/branches/_branch.html.haml' do - element :remove_btn element :branch_name end diff --git a/qa/qa/page/project/members.rb b/qa/qa/page/project/members.rb index eeb589d6ca8..1102abd6646 100644 --- a/qa/qa/page/project/members.rb +++ b/qa/qa/page/project/members.rb @@ -41,6 +41,11 @@ module QA click_button 'Remove group' end end + + def has_group?(group_name) + click_element :groups_list_tab + has_element?(:group_row, text: group_name) + end end end end diff --git a/qa/qa/page/project/new.rb b/qa/qa/page/project/new.rb index 5ff52527774..42baf1f3f87 100644 --- a/qa/qa/page/project/new.rb +++ b/qa/qa/page/project/new.rb @@ -14,7 +14,6 @@ module QA view 'app/views/projects/_new_project_fields.html.haml' do element :initialize_with_readme_checkbox element :initialize_with_sast_checkbox - element :project_namespace_field, 'namespaces_options' # rubocop:disable QA/ElementWithPattern element :project_name, 'text_field :name' # rubocop:disable QA/ElementWithPattern element :project_path, 'text_field :path' # rubocop:disable QA/ElementWithPattern element :project_description, 'text_area :description' # rubocop:disable QA/ElementWithPattern diff --git a/qa/qa/page/project/packages/show.rb b/qa/qa/page/project/packages/show.rb index 4872c0bc705..5ba9ad7df40 100644 --- a/qa/qa/page/project/packages/show.rb +++ b/qa/qa/page/project/packages/show.rb @@ -5,7 +5,7 @@ module QA module Project module Packages class Show < QA::Page::Base - view 'app/assets/javascripts/packages_and_registries/package_registry/components/details/app.vue' do + view 'app/assets/javascripts/packages_and_registries/package_registry/pages/details.vue' do element :delete_button element :delete_modal_button element :package_information_content diff --git a/qa/qa/page/project/pipeline_editor/show.rb b/qa/qa/page/project/pipeline_editor/show.rb index e430884ea08..8289039d4c5 100644 --- a/qa/qa/page/project/pipeline_editor/show.rb +++ b/qa/qa/page/project/pipeline_editor/show.rb @@ -24,6 +24,27 @@ module QA element :source_editor_container, require: true end + view 'app/assets/javascripts/pipeline_editor/components/header/pipeline_status.vue' do + element :pipeline_id_content + end + + view 'app/assets/javascripts/pipeline_editor/components/commit/commit_form.vue' do + element :commit_changes_button + end + + view 'app/assets/javascripts/pipeline_editor/components/header/validation_segment.vue' do + element :validation_message_content + end + + view 'app/assets/javascripts/pipelines/components/pipeline_graph/pipeline_graph.vue' do + element :stage_container + element :job_container + end + + view 'app/assets/javascripts/pipeline_editor/components/pipeline_editor_tabs.vue' do + element :file_editor_container + end + def initialize super @@ -50,8 +71,70 @@ module QA find_element(:source_editor_container).text end + def write_to_editor(text) + find_element(:source_editor_container).fill_in(with: text) + end + + def submit_changes + click_element(:commit_changes_button) + + wait_for_requests + end + + def set_target_branch(name) + find_element(:target_branch_field).fill_in(with: name) + end + + def current_branch + find_element(:branch_selector_button).text + end + + def pipeline_id + find_element(:pipeline_id_content).text.delete!('#').to_i + end + + def ci_syntax_validate_message + find_element(:validation_message_content).text + end + + def go_to_visualize_tab + go_to_tab('Visualize') + end + + def go_to_lint_tab + go_to_tab('Lint') + end + + def go_to_view_merged_yaml_tab + go_to_tab('View merged YAML') + end + + def has_source_editor? + has_element?(:source_editor_container) + end + + def has_stage?(name) + all_elements(:stage_container, minimum: 1).any? { |item| item.text.match(/#{name}/i) } + end + + def has_job?(name) + all_elements(:job_container, minimum: 1).any? { |item| item.text.match(/#{name}/i) } + end + + def tab_alert_message + within_element(:file_editor_container) do + find('.gl-alert-body').text + end + end + private + def go_to_tab(name) + within_element(:file_editor_container) do + find('.nav-item', text: name).click + end + end + # If the page thinks user has never opened pipeline editor before # It will expand pipeline editor sidebar by default # Collapse the sidebar if it is expanded diff --git a/qa/qa/page/project/secure/configuration_form.rb b/qa/qa/page/project/secure/configuration_form.rb index 3e89a57e870..fa1fad44273 100644 --- a/qa/qa/page/project/secure/configuration_form.rb +++ b/qa/qa/page/project/secure/configuration_form.rb @@ -8,6 +8,10 @@ module QA include QA::Page::Component::Select2 include QA::Page::Settings::Common + view 'app/assets/javascripts/security_configuration/components/app.vue' do + element :security_configuration_history_link + end + view 'app/assets/javascripts/security_configuration/components/feature_card.vue' do element :dependency_scanning_status, "`${feature.type}_status`" # rubocop:disable QA/ElementWithPattern element :sast_status, "`${feature.type}_status`" # rubocop:disable QA/ElementWithPattern @@ -15,6 +19,22 @@ module QA element :dependency_scanning_mr_button, "`${feature.type}_mr_button`" # rubocop:disable QA/ElementWithPattern end + view 'app/assets/javascripts/security_configuration/components/auto_dev_ops_alert.vue' do + element :autodevops_container + end + + def has_security_configuration_history_link? + has_element?(:security_configuration_history_link) + end + + def has_no_security_configuration_history_link? + has_no_element?(:security_configuration_history_link) + end + + def click_security_configuration_history_link + click_element(:security_configuration_history_link) + end + def click_sast_enable_button click_element(:sast_enable_button) end @@ -29,11 +49,37 @@ module QA end end + def has_no_sast_status?(status_text) + within_element(:sast_status) do + has_no_text?(status_text) + end + end + def has_dependency_scanning_status?(status_text) within_element(:dependency_scanning_status) do has_text?(status_text) end end + + def has_no_dependency_scanning_status?(status_text) + within_element(:dependency_scanning_status) do + has_no_text?(status_text) + end + end + + def has_auto_devops_container? + has_element?(:autodevops_container) + end + + def has_no_auto_devops_container? + has_no_element?(:autodevops_container) + end + + def has_auto_devops_container_description? + within_element(:autodevops_container) do + has_text?('Quickly enable all continuous testing and compliance tools by enabling Auto DevOps') + end + end end end end diff --git a/qa/qa/page/project/settings/visibility_features_permissions.rb b/qa/qa/page/project/settings/visibility_features_permissions.rb index 1d6686ae360..60cea6de7f5 100644 --- a/qa/qa/page/project/settings/visibility_features_permissions.rb +++ b/qa/qa/page/project/settings/visibility_features_permissions.rb @@ -5,14 +5,9 @@ module QA module Project module Settings class VisibilityFeaturesPermissions < Page::Base - include QA::Page::Component::Select2 - - view 'app/views/projects/edit.html.haml' do - element :visibility_features_permissions_save_button - end - view 'app/assets/javascripts/pages/projects/shared/permissions/components/settings_panel.vue' do element :project_visibility_dropdown + element :visibility_features_permissions_save_button end def set_project_visibility(visibility) diff --git a/qa/qa/page/registration/sign_up.rb b/qa/qa/page/registration/sign_up.rb index 6d1b9cb3615..4fedc05c702 100644 --- a/qa/qa/page/registration/sign_up.rb +++ b/qa/qa/page/registration/sign_up.rb @@ -16,10 +16,6 @@ module QA element :new_user_username_field end - view 'app/views/registrations/welcome/show.html.haml' do - element :get_started_button - end - def fill_new_user_first_name_field(first_name) fill_element :new_user_first_name_field, first_name end diff --git a/qa/qa/page/registration/welcome.rb b/qa/qa/page/registration/welcome.rb index ff22e62b63e..660b33b4f5b 100644 --- a/qa/qa/page/registration/welcome.rb +++ b/qa/qa/page/registration/welcome.rb @@ -6,14 +6,25 @@ module QA class Welcome < Page::Base view 'app/views/registrations/welcome/show.html.haml' do element :get_started_button + element :role_dropdown end - def click_get_started_button_if_available - if has_element?(:get_started_button) - Support::Retrier.retry_until do - click_element :get_started_button - has_no_element?(:get_started_button) - end + def has_get_started_button? + has_element?(:get_started_button) + end + + def select_role(role) + select_element(:role_dropdown, role) + end + + def choose_setup_for_company_if_available + # Only implemented in EE + end + + def click_get_started_button + Support::Retrier.retry_until do + click_element :get_started_button + has_no_element?(:get_started_button) end end end diff --git a/qa/qa/page/trials/new.rb b/qa/qa/page/trials/new.rb index 268f3b71717..6e9d7fce688 100644 --- a/qa/qa/page/trials/new.rb +++ b/qa/qa/page/trials/new.rb @@ -6,17 +6,13 @@ module QA class New < Chemlab::Page path '/-/trials/new' - # TODO: Supplant with data-qa-selectors - text_field :first_name, id: 'first_name' - text_field :last_name, id: 'last_name' - text_field :company_name, id: 'company_name' - select :number_of_employees, id: 'company_size' - text_field :telephone_number, id: 'phone_number' - text_field :number_of_users, id: 'number_of_users' - - select :country, id: 'country_select' - - button :continue, value: 'Continue' + text_field :first_name + text_field :last_name + text_field :company_name + select :number_of_employees + text_field :telephone_number + select :country + button :continue end end end diff --git a/qa/qa/resource/api_fabricator.rb b/qa/qa/resource/api_fabricator.rb index 6315ef0bd22..4c77c515cfd 100644 --- a/qa/qa/resource/api_fabricator.rb +++ b/qa/qa/resource/api_fabricator.rb @@ -63,6 +63,10 @@ module QA process_api_response(parse_body(response)) end + def api_fabrication_http_method + @api_fabrication_http_method ||= :post + end + private def resource_web_url(resource) @@ -85,6 +89,8 @@ module QA raise ResourceNotFoundError, "Resource at #{request.mask_url} could not be found (#{response.code}): `#{response}`." end + @api_fabrication_http_method = :get # rubocop:disable Gitlab/ModuleWithInstanceVariables + response end diff --git a/qa/qa/resource/base.rb b/qa/qa/resource/base.rb index 640d2a8f06e..0112e766cf0 100644 --- a/qa/qa/resource/base.rb +++ b/qa/qa/resource/base.rb @@ -71,34 +71,49 @@ module QA resource_web_url = yield resource.web_url = resource_web_url + QA::Tools::TestResourceDataProcessor.collect(resource, resource_identifier(resource)) + resource end + def resource_identifier(resource) + if resource.respond_to?(:username) && resource.username + "with username '#{resource.username}'" + elsif resource.respond_to?(:full_path) && resource.full_path + "with full_path '#{resource.full_path}'" + elsif resource.respond_to?(:name) && resource.name + "with name '#{resource.name}'" + elsif resource.respond_to?(:id) && resource.id + "with id '#{resource.id}'" + elsif resource.respond_to?(:iid) && resource.iid + "with iid '#{resource.iid}'" + end + rescue QA::Resource::Base::NoValueError + nil + end + def log_fabrication(method, resource, parents, args) start = Time.now Support::FabricationTracker.start_fabrication result = yield.tap do fabrication_time = Time.now - start - resource_identifier = begin - if resource.respond_to?(:username) && resource.username - "with username '#{resource.username}'" - elsif resource.respond_to?(:full_path) && resource.full_path - "with full_path '#{resource.full_path}'" - elsif resource.respond_to?(:name) && resource.name - "with name '#{resource.name}'" - elsif resource.respond_to?(:id) && resource.id - "with id '#{resource.id}'" - end - rescue QA::Resource::Base::NoValueError - nil - end + + fabrication_http_method = if resource.api_fabrication_http_method == :get + if self.include?(Reusable) + "Retrieved for reuse" + else + "Retrieved" + end + else + "Built" + end Support::FabricationTracker.save_fabrication(:"#{method}_fabrication", fabrication_time) Runtime::Logger.debug do msg = ["==#{'=' * parents.size}>"] - msg << "Built a #{name}" - msg << resource_identifier if resource_identifier + msg << "#{fabrication_http_method} a #{name}" + msg << resource_identifier(resource) if resource_identifier(resource) msg << "as a dependency of #{parents.last}" if parents.any? msg << "via #{method}" msg << "in #{fabrication_time} seconds" @@ -156,11 +171,11 @@ module QA raise NotImplementedError end - def visit! + def visit!(skip_resp_code_check: false) Runtime::Logger.debug(%(Visiting #{self.class.name} at "#{web_url}")) # Just in case an async action is not yet complete - Support::WaitForRequests.wait_for_requests + Support::WaitForRequests.wait_for_requests(skip_resp_code_check: skip_resp_code_check) Support::Retrier.retry_until do visit(web_url) @@ -168,7 +183,7 @@ module QA end # Wait until the new page is ready for us to interact with it - Support::WaitForRequests.wait_for_requests + Support::WaitForRequests.wait_for_requests(skip_resp_code_check: skip_resp_code_check) end def populate(*attribute_names) @@ -179,6 +194,30 @@ module QA QA::Support::Waiter.wait_until(max_duration: max_duration, sleep_interval: sleep_interval, &block) end + # Object comparison + # + # @param [QA::Resource::Base] other + # @return [Boolean] + def ==(other) + other.is_a?(self.class) && comparable == other.comparable + end + + # Override inspect for a better rspec failure diff output + # + # @return [String] + def inspect + JSON.pretty_generate(comparable) + end + + protected + + # Custom resource comparison logic using resource attributes from api_resource + # + # @return [Hash] + def comparable + raise("comparable method needs to be implemented in order to compare resources via '=='") + end + private def attribute_value(name, block) diff --git a/qa/qa/resource/bulk_import_group.rb b/qa/qa/resource/bulk_import_group.rb index e8dc2d291b8..a22529152e1 100644 --- a/qa/qa/resource/bulk_import_group.rb +++ b/qa/qa/resource/bulk_import_group.rb @@ -3,18 +3,21 @@ module QA module Resource class BulkImportGroup < Group - attributes :source_group_path, + attributes :source_group, + :destination_group, :import_id - attribute :destination_group_path do - source_group_path - end - attribute :access_token do api_client.personal_access_token end - alias_method :path, :source_group_path + # In most cases we will want to set path the same as source group + # but it can be set to a custom name as well when imported via API + attribute :destination_group_path do + source_group.path + end + # Can't define path as attribue since @path is set in base class initializer + alias_method :path, :destination_group_path delegate :gitlab_address, to: 'QA::Runtime::Scenario' @@ -51,9 +54,9 @@ module QA entities: [ { source_type: 'group_entity', - source_full_path: source_group_path, + source_full_path: source_group.full_path, destination_name: destination_group_path, - destination_namespace: sandbox.path + destination_namespace: sandbox.full_path } ] } diff --git a/qa/qa/resource/group_badge.rb b/qa/qa/resource/group_badge.rb index fd76f066e8b..3719b502b93 100644 --- a/qa/qa/resource/group_badge.rb +++ b/qa/qa/resource/group_badge.rb @@ -39,27 +39,12 @@ module QA # @return [String] def resource_web_url(_resource); end - # Object comparison - # - # @param [QA::Resource::GroupBadge] other - # @return [Boolean] - def ==(other) - other.is_a?(GroupBadge) && comparable_badge == other.comparable_badge - end - - # Override inspect for a better rspec failure diff output - # - # @return [String] - def inspect - JSON.pretty_generate(comparable_badge) - end - protected # Return subset of fields for comparing badges # # @return [Hash] - def comparable_badge + def comparable reload! unless api_response api_response.slice( diff --git a/qa/qa/resource/group_base.rb b/qa/qa/resource/group_base.rb index 19bb5ea00d7..9f492a046db 100644 --- a/qa/qa/resource/group_base.rb +++ b/qa/qa/resource/group_base.rb @@ -123,18 +123,12 @@ module QA end # Object comparison + # Override to make sure we are comparing descendands of GroupBase # # @param [QA::Resource::GroupBase] other # @return [Boolean] def ==(other) - other.is_a?(GroupBase) && comparable_group == other.comparable_group - end - - # Override inspect for a better rspec failure diff output - # - # @return [String] - def inspect - JSON.pretty_generate(comparable_group) + other.is_a?(GroupBase) && comparable == other.comparable end protected @@ -142,7 +136,7 @@ module QA # Return subset of fields for comparing groups # # @return [Hash] - def comparable_group + def comparable reload! if api_response.nil? api_resource.slice( diff --git a/qa/qa/resource/group_milestone.rb b/qa/qa/resource/group_milestone.rb index 880ca2b9721..b9ec53e929c 100644 --- a/qa/qa/resource/group_milestone.rb +++ b/qa/qa/resource/group_milestone.rb @@ -56,27 +56,12 @@ module QA end end - # Object comparison - # - # @param [QA::Resource::GroupMilestone] other - # @return [Boolean] - def ==(other) - other.is_a?(GroupMilestone) && comparable_milestone == other.comparable_milestone - end - - # Override inspect for a better rspec failure diff output - # - # @return [String] - def inspect - JSON.pretty_generate(comparable_milestone) - end - protected # Return subset of fields for comparing milestones # # @return [Hash] - def comparable_milestone + def comparable reload! unless api_response api_response.slice( diff --git a/qa/qa/resource/issue.rb b/qa/qa/resource/issue.rb index 344f177932f..1e38de97c1e 100644 --- a/qa/qa/resource/issue.rb +++ b/qa/qa/resource/issue.rb @@ -3,7 +3,7 @@ module QA module Resource class Issue < Base - attr_writer :description, :milestone, :template, :weight + attr_writer :milestone, :template, :weight attribute :project do Project.fabricate! do |resource| @@ -95,19 +95,13 @@ module QA ) end - # Object comparison + # Create a new comment # - # @param [QA::Resource::Issue] other - # @return [Boolean] - def ==(other) - other.is_a?(Issue) && comparable_issue == other.comparable_issue - end - - # Override inspect for a better rspec failure diff output - # - # @return [String] - def inspect - JSON.pretty_generate(comparable_issue) + # @param [String] body + # @param [Boolean] confidential + # @return [Hash] + def add_comment(body:, confidential: false) + api_post_to(api_comments_path, body: body, confidential: confidential) end protected @@ -115,7 +109,7 @@ module QA # Return subset of fields for comparing issues # # @return [Hash] - def comparable_issue + def comparable reload! if api_response.nil? api_resource.slice( diff --git a/qa/qa/resource/label_base.rb b/qa/qa/resource/label_base.rb index b1af0e23561..855e41af8bb 100644 --- a/qa/qa/resource/label_base.rb +++ b/qa/qa/resource/label_base.rb @@ -49,27 +49,12 @@ module QA } end - # Object comparison - # - # @param [QA::Resource::GroupBase] other - # @return [Boolean] - def ==(other) - other.is_a?(LabelBase) && comparable_label == other.comparable_label - end - - # Override inspect for a better rspec failure diff output - # - # @return [String] - def inspect - JSON.pretty_generate(comparable_label) - end - protected # Return subset of fields for comparing labels # # @return [Hash] - def comparable_label + def comparable reload! unless api_response api_response.slice( diff --git a/qa/qa/resource/merge_request.rb b/qa/qa/resource/merge_request.rb index ba63e0823f0..685cccea02d 100644 --- a/qa/qa/resource/merge_request.rb +++ b/qa/qa/resource/merge_request.rb @@ -6,11 +6,13 @@ module QA attr_accessor :approval_rules, :source_branch, :target_new_branch, + :update_existing_file, :assignee, :milestone, :labels, :file_name, :file_content + attr_writer :no_preparation, :wait_for_merge, :template @@ -25,6 +27,8 @@ module QA attribute :project do Project.fabricate_via_api! do |resource| resource.name = 'project-with-merge-request' + resource.initialize_with_readme = true + resource.api_client = api_client end end @@ -33,22 +37,27 @@ module QA end attribute :target do - Repository::ProjectPush.fabricate! do |resource| + Repository::Commit.fabricate_via_api! do |resource| resource.project = project - resource.branch_name = target_branch - resource.new_branch = target_new_branch - resource.remote_branch = target_branch + resource.api_client = api_client + resource.commit_message = 'This is a test commit' + resource.add_files([{ 'file_path': "file-#{SecureRandom.hex(8)}.txt", 'content': 'MR init' }]) + resource.branch = target_branch + + resource.start_branch = project.default_branch if target_branch != project.default_branch end end attribute :source do - Repository::ProjectPush.fabricate! do |resource| + Repository::Commit.fabricate_via_api! do |resource| resource.project = project - resource.branch_name = target_branch - resource.remote_branch = source_branch - resource.new_branch = false - resource.file_name = file_name - resource.file_content = file_content + resource.api_client = api_client + resource.commit_message = 'This is a test commit' + resource.branch = source_branch + resource.start_branch = target_branch + + files = [{ 'file_path': file_name, 'content': file_content }] + update_existing_file ? resource.update_files(files) : resource.add_files(files) end end @@ -63,6 +72,7 @@ module QA @file_name = "added_file-#{SecureRandom.hex(8)}.txt" @file_content = "File Added" @target_new_branch = true + @update_existing_file = false @no_preparation = false @wait_for_merge = true end @@ -168,27 +178,18 @@ module QA ) end - # Object comparison - # - # @param [QA::Resource::MergeRequest] other - # @return [Boolean] - def ==(other) - other.is_a?(MergeRequest) && comparable_mr == other.comparable_mr - end - - # Override inspect for a better rspec failure diff output + # Add mr comment # - # @return [String] - def inspect - JSON.pretty_generate(comparable_mr) + # @param [String] body + # @return [Hash] + def add_comment(body) + api_post_to(api_comments_path, body: body) end - protected - # Return subset of fields for comparing merge requests # # @return [Hash] - def comparable_mr + def comparable reload! if api_response.nil? api_resource.except( @@ -197,7 +198,9 @@ module QA :project_id, :source_project_id, :target_project_id, + :merge_status, # these can differ depending on user fetching mr + :user, :subscribed, :first_contribution ).merge({ references: api_resource[:references].except(:full) }) @@ -211,8 +214,24 @@ module QA super(api_resource) end + # Create source and target and commits if necessary + # + # @return [void] def populate_target_and_source_if_required - populate(:target, :source) unless @no_preparation + return if @no_preparation + + populate(:target) if create_target? + populate(:source) + end + + # Check if target needs to be created + # + # Return false if project was already initialized and mr target is default branch + # Return false if target_new_branch is explicitly set to false + # + # @return [Boolean] + def create_target? + !(project.initialize_with_readme && target_branch == project.default_branch) && target_new_branch end end end diff --git a/qa/qa/resource/project.rb b/qa/qa/resource/project.rb index d3c1e91f358..0750ea49224 100644 --- a/qa/qa/resource/project.rb +++ b/qa/qa/resource/project.rb @@ -107,32 +107,6 @@ module QA super end - def has_file?(file_path) - response = repository_tree - - raise ResourceNotFoundError, (response[:message]).to_s if response.is_a?(Hash) && response.has_key?(:message) - - response.any? { |file| file[:path] == file_path } - end - - def has_branch?(branch) - has_branches?(Array(branch)) - end - - def has_branches?(branches) - branches.all? do |branch| - response = get(request_url("#{api_repository_branches_path}/#{branch}")) - response.code == HTTP_STATUS_OK - end - end - - def has_tags?(tags) - tags.all? do |tag| - response = get(request_url("#{api_repository_tags_path}/#{tag}")) - response.code == HTTP_STATUS_OK - end - end - def api_get_path "/projects/#{CGI.escape(path_with_namespace)}" end @@ -237,6 +211,32 @@ module QA post_body end + def has_file?(file_path) + response = repository_tree + + raise ResourceNotFoundError, (response[:message]).to_s if response.is_a?(Hash) && response.has_key?(:message) + + response.any? { |file| file[:path] == file_path } + end + + def has_branch?(branch) + has_branches?(Array(branch)) + end + + def has_branches?(branches) + branches.all? do |branch| + response = get(request_url("#{api_repository_branches_path}/#{branch}")) + response.code == HTTP_STATUS_OK + end + end + + def has_tags?(tags) + tags.all? do |tag| + response = get(request_url("#{api_repository_tags_path}/#{tag}")) + response.code == HTTP_STATUS_OK + end + end + def change_repository_storage(new_storage) response = put(request_url(api_put_path), repository_storage: new_storage) @@ -372,27 +372,12 @@ module QA api_post_to(api_wikis_path, title: title, content: content) end - # Object comparison - # - # @param [QA::Resource::Project] other - # @return [Boolean] - def ==(other) - other.is_a?(Project) && comparable_project == other.comparable_project - end - - # Override inspect for a better rspec failure diff output - # - # @return [String] - def inspect - JSON.pretty_generate(comparable_project) - end - protected # Return subset of fields for comparing projects # # @return [Hash] - def comparable_project + def comparable reload! if api_response.nil? api_resource.slice( diff --git a/qa/qa/resource/repository/commit.rb b/qa/qa/resource/repository/commit.rb index f5dba164de6..54093a5c195 100644 --- a/qa/qa/resource/repository/commit.rb +++ b/qa/qa/resource/repository/commit.rb @@ -22,42 +22,7 @@ module QA def initialize @commit_message = 'QA Test - Commit message' - end - - def add_files(files) - validate_files!(files) - - @add_files = files - end - - def add_directory(dir) - raise "Must set directory as a Pathname" unless dir.is_a?(Pathname) - - files_to_add = [] - - dir.each_child do |child| - case child.ftype? - when "file" - files_to_add.append({ - file_path: child.to_s, - content: child.read - }) - when "directory" - add_directory(child) - else - continue - end - end - - validate_files!(files_to_add) - - @add_files.merge(files_to_add) - end - - def update_files(files) - validate_files!(files) - - @update_files = files + @actions = [] end # If `actions` are specified, it performs the actions to create, @@ -72,32 +37,74 @@ module QA end def api_get_path - api_post_path + "/projects/#{CGI.escape(project.path_with_namespace)}/repository/commits" end def api_post_path - "/projects/#{CGI.escape(project.path_with_namespace)}/repository/commits" + api_get_path end def api_post_body { - branch: @branch || project.default_branch, - author_email: @author_email || Runtime::User.default_email, - author_name: @author_name || Runtime::User.username, + branch: branch || project.default_branch, + author_email: author_email || api_client.user&.email || Runtime::User.default_email, + author_name: author_name || api_client.user&.name || Runtime::User.username, commit_message: commit_message, actions: actions }.merge(new_branch) end - def actions - pending_actions = [] - pending_actions << @add_files.map { |file| file.merge({ action: "create" }) } if @add_files - pending_actions << @update_files.map { |file| file.merge({ action: "update" }) } if @update_files - pending_actions.flatten + # Add files + # Pass in array of new files like, example: + # [{ "file_path": "foo/bar", "content": "some content" }] + # + # @param [Array<Hash>] files + # @return [void] + def add_files(files) + validate_files!(files) + + actions.push(*files.map { |file| file.merge({ action: "create" }) }) + end + + # Update files + # Pass in array of files and it's contents, example: + # [{ "file_path": "foo/bar", "content": "some content" }] + # + # @param [Array<Hash>] files + # @return [void] + def update_files(files) + validate_files!(files) + + actions.push(*files.map { |file| file.merge({ action: "update" }) }) + end + + # Add all files from directory + # + # @param [Pathname] dir + # @return [void] + def add_directory(dir) + raise "Must set directory as a Pathname" unless dir.is_a?(Pathname) + + files_to_add = [] + + dir.each_child do |child| + case child.ftype + when "directory" + add_directory(child) + when "file" + files_to_add.push({ file_path: child.basename, content: child.read }) + else + continue + end + end + + add_files(files_to_add) end private + attr_reader :actions + def validate_files!(files) if !files.is_a?(Array) || files.empty? || diff --git a/qa/qa/resource/reusable_group.rb b/qa/qa/resource/reusable_group.rb new file mode 100644 index 00000000000..a4bd799e85c --- /dev/null +++ b/qa/qa/resource/reusable_group.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module QA + module Resource + class ReusableGroup < Group + prepend Reusable + + def initialize + super + + @path = "reusable_group" + @description = "QA reusable group" + @reuse_as = :default_group + end + + # Confirms that the group can be reused + # + # @return [nil] returns nil unless an error is raised + def validate_reuse_preconditions + unless reused_path_unique? + raise ResourceReuseError, + "Reusable groups must have the same name. The group reused as #{reuse_as} has the path '#{path}' but it should be '#{self.class.resources[reuse_as].path}'" + end + end + + # Confirms that reuse of the resource did not change it in a way that breaks later reuse. This raises an error if + # the current group path doesn't match the original path. + def validate_reuse + reload! + + if api_resource[:path] != @path + raise ResourceReuseError, "The group now has the path '#{api_resource[:path]}' but it should be '#{path}'" + end + end + + # Checks if the group is being reused with the same path. + # + # @return [Boolean] true if the group's path is different from another group with the same reuse symbol (reuse_as) + def reused_path_unique? + return true unless self.class.resources.key?(reuse_as) + + self.class.resources[reuse_as].path == path + end + + # Overrides QA::Resource::Group#remove_via_api! to log a debug message stating that removal will happen after + # the suite completes rather than now. + # + # @return [nil] + def remove_via_api! + QA::Runtime::Logger.debug("#{self.class.name} - deferring removal until after suite") + end + end + end +end diff --git a/qa/qa/resource/reusable_project.rb b/qa/qa/resource/reusable_project.rb index 6b33bb9d65d..d2dfff8ad56 100644 --- a/qa/qa/resource/reusable_project.rb +++ b/qa/qa/resource/reusable_project.rb @@ -5,6 +5,12 @@ module QA class ReusableProject < Project prepend Reusable + attribute :group do + ReusableGroup.fabricate_via_api! do |resource| + resource.api_client = api_client + end + end + def initialize super diff --git a/qa/qa/resource/user.rb b/qa/qa/resource/user.rb index ed4ea057484..eab428a61e7 100644 --- a/qa/qa/resource/user.rb +++ b/qa/qa/resource/user.rb @@ -98,6 +98,12 @@ module QA super end + def exists? + api_get + rescue ResourceNotFoundError + false + end + def api_delete super @@ -181,6 +187,15 @@ module QA ) end + protected + + # Compare users by username and password + # + # @return [Array] + def comparable + [username, password] + end + private def ldap_post_body diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb index b73199f1fdd..1679698a9c0 100644 --- a/qa/qa/runtime/env.rb +++ b/qa/qa/runtime/env.rb @@ -9,6 +9,7 @@ module QA extend self attr_writer :personal_access_token, :admin_personal_access_token + attr_accessor :dry_run ENV_VARIABLES = Gitlab::QA::Runtime::Env::ENV_VARIABLES @@ -89,6 +90,20 @@ module QA enabled?(ENV['ACCEPT_INSECURE_CERTS']) end + def running_on_dot_com? + uri = URI.parse(Runtime::Scenario.gitlab_address) + uri.host.include?('.com') + end + + def running_on_dev? + uri = URI.parse(Runtime::Scenario.gitlab_address) + uri.port != 80 && uri.port != 443 + end + + def running_on_dev_or_dot_com? + running_on_dev? || running_on_dot_com? + end + def running_in_ci? ENV['CI'] || ENV['CI_SERVER'] end @@ -281,9 +296,7 @@ module QA end def knapsack? - return false unless ENV['CI_NODE_TOTAL'].to_i > 1 - - !!(ENV['KNAPSACK_GENERATE_REPORT'] || ENV['KNAPSACK_REPORT_PATH'] || ENV['KNAPSACK_TEST_FILE_PATTERN']) + ENV['CI_NODE_TOTAL'].to_i > 1 && ENV['NO_KNAPSACK'] != "true" end def ldap_username @@ -401,7 +414,7 @@ module QA end def gitlab_agentk_version - ENV.fetch('GITLAB_AGENTK_VERSION', 'v14.4.0') + ENV.fetch('GITLAB_AGENTK_VERSION', 'v14.5.0') end def transient_trials @@ -416,6 +429,11 @@ module QA running_in_ci? && enabled?(ENV['QA_EXPORT_TEST_METRICS'], default: true) end + def test_resources_created_filepath + file_name = running_in_ci? ? "test-resources-#{SecureRandom.hex(3)}.json" : 'test-resources.json' + ENV.fetch('QA_TEST_RESOURCES_CREATED_FILEPATH', File.join(Path.qa_root, 'tmp', file_name)) + end + private def remote_grid_credentials diff --git a/qa/qa/scenario/bootable.rb b/qa/qa/scenario/bootable.rb index ae180ffce1c..2a9bbbc9fdb 100644 --- a/qa/qa/scenario/bootable.rb +++ b/qa/qa/scenario/bootable.rb @@ -31,6 +31,13 @@ module QA end next + elsif opt.name == :count_examples_only + parser.on(opt.arg, opt.desc) do |value| + QA::Runtime::Env.dry_run = true + Runtime::Scenario.define(opt.name, value) + end + + next end parser.on(opt.arg, opt.desc) do |value| diff --git a/qa/qa/scenario/shared_attributes.rb b/qa/qa/scenario/shared_attributes.rb index ddbe28f05d9..d5d7aedb47f 100644 --- a/qa/qa/scenario/shared_attributes.rb +++ b/qa/qa/scenario/shared_attributes.rb @@ -13,6 +13,7 @@ module QA 'Specify FEATURE_FLAGS as comma-separated flag=state pairs, e.g., "flag1=enabled,flag2=disabled"' attribute :parallel, '--parallel', 'Execute tests in parallel' attribute :loop, '--loop', 'Execute test repeatedly' + attribute :count_examples_only, '--count-examples-only', 'Return the number of examples without running them' end end end diff --git a/qa/qa/scenario/template.rb b/qa/qa/scenario/template.rb index 50bb952f1fd..ef634d3ccda 100644 --- a/qa/qa/scenario/template.rb +++ b/qa/qa/scenario/template.rb @@ -33,9 +33,15 @@ module QA Runtime::Scenario.define(:about_address, URI(-> { gitlab_address.host = "about.#{gitlab_address.host}"; gitlab_address }.call).to_s) # rubocop:disable Style/Semicolon ## + # Setup knapsack and download latest report + # + Tools::KnapsackReport.configure! if Runtime::Env.knapsack? + + ## # Perform before hooks, which are different for CE and EE # - Runtime::Release.perform_before_hooks + + Runtime::Release.perform_before_hooks unless Runtime::Env.dry_run Runtime::Feature.enable(options[:enable_feature]) if options.key?(:enable_feature) Runtime::Feature.disable(options[:disable_feature]) if options.key?(:disable_feature) && (@feature_enabled = Runtime::Feature.enabled?(options[:disable_feature])) diff --git a/qa/qa/service/praefect_manager.rb b/qa/qa/service/praefect_manager.rb index dd4cce5d0b0..7e47049d446 100644 --- a/qa/qa/service/praefect_manager.rb +++ b/qa/qa/service/praefect_manager.rb @@ -19,7 +19,7 @@ module QA @virtual_storage = 'default' end - attr_reader :primary_node, :secondary_node, :tertiary_node + attr_reader :primary_node, :secondary_node, :tertiary_node, :postgres # Executes the praefect `dataloss` command. # @@ -83,19 +83,19 @@ module QA def start_node(name) shell "docker start #{name}" + end + + def stop_node(name) + shell "docker stop #{name}" wait_until_shell_command_matches( "docker inspect -f {{.State.Running}} #{name}", - /true/, + /false/, sleep_interval: 3, max_duration: 180, retry_on_exception: true ) end - def stop_node(name) - shell "docker stop #{name}" - end - def clear_replication_queue QA::Runtime::Logger.info("Clearing the replication queue") shell sql_to_docker_exec_cmd( @@ -174,13 +174,13 @@ module QA end def start_all_nodes + start_node(@postgres) start_node(@primary_node) start_node(@secondary_node) start_node(@tertiary_node) start_node(@praefect) wait_for_health_check_all_nodes - wait_for_reliable_connection end def verify_storage_move(source_storage, destination_storage, repo_type: :project) @@ -192,21 +192,23 @@ module QA end def wait_for_praefect - wait_until_shell_command_matches( - "docker inspect -f {{.State.Running}} #{@praefect}", - /true/, - sleep_interval: 3, - max_duration: 180, - retry_on_exception: true - ) + QA::Runtime::Logger.info("Waiting for health check on praefect") + Support::Waiter.wait_until(max_duration: 120, sleep_interval: 1, raise_on_failure: true) do + # praefect runs a grpc server on port 2305, which will return an error 'Connection refused' until such time it is ready + wait_until_shell_command("docker exec #{@gitaly_cluster} bash -c 'curl #{@praefect}:2305'") do |line| + break if line.include?('curl: (1) Received HTTP/0.9 when not allowed') - QA::Runtime::Logger.info('Wait until Praefect starts and is listening') - wait_until_shell_command_matches( - "docker exec #{@praefect} bash -c 'cat /var/log/gitlab/praefect/current'", - /listening at tcp address/ - ) + QA::Runtime::Logger.debug(line.chomp) + end + end + end - wait_for_gitaly_check + def praefect_sql_ping_healthy? + cmd = "docker exec #{@praefect} bash -c '/opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml sql-ping'" + wait_until_shell_command(cmd) do |line| + QA::Runtime::Logger.debug(line.chomp) + break line.include?('praefect sql-ping: OK') + end end def wait_for_sql_ping @@ -220,32 +222,7 @@ module QA ['error when pinging healthcheck', 'failed checking node health'].include?(msg) end - def wait_for_no_praefect_storage_error - # If a healthcheck error was the last message to be logged, we'll keep seeing that message even if it's no longer a problem - # That is, there's no message shown in the Praefect logs when the healthcheck succeeds - # To work around that we perform the gitaly check rake task, wait a few seconds, and then we confirm that no healthcheck errors appear - - QA::Runtime::Logger.info("Checking that Praefect does not report healthcheck errors with its gitaly nodes") - - Support::Waiter.wait_until(max_duration: 120) do - wait_for_gitaly_check - - sleep 5 - - shell "docker exec #{@praefect} bash -c 'tail -n 1 /var/log/gitlab/praefect/current'" do |line| - QA::Runtime::Logger.debug(line.chomp) - log = JSON.parse(line) - - break true unless health_check_failure_message?(log['msg']) - rescue JSON::ParserError - # Ignore lines that can't be parsed as JSON - end - end - end - - def wait_for_storage_nodes - wait_for_no_praefect_storage_error - + def wait_for_dial_nodes_successful Support::Waiter.repeat_until(max_attempts: 3, max_duration: 120, sleep_interval: 1) do nodes_confirmed = { @primary_node => false, @@ -253,39 +230,55 @@ module QA @tertiary_node => false } - wait_until_shell_command("docker exec #{@praefect} bash -c '/opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dial-nodes'") do |line| - QA::Runtime::Logger.debug(line.chomp) + nodes_confirmed.each_key do |node| + nodes_confirmed[node] = true if praefect_dial_nodes_status?(node) + end - nodes_confirmed.each_key do |node| - nodes_confirmed[node] = true if line =~ /SUCCESS: confirmed Gitaly storage "#{node}" in virtual storages \[#{@virtual_storage}\] is served/ - end + nodes_confirmed.values.all? + end + end - nodes_confirmed.values.all? + def praefect_dial_nodes_status?(node, expect_healthy = true) + cmd = "docker exec #{@praefect} bash -c '/opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dial-nodes -timeout 1s'" + if expect_healthy + wait_until_shell_command_matches(cmd, /SUCCESS: confirmed Gitaly storage "#{node}" in virtual storages \[#{@virtual_storage}\] is served/) + else + wait_until_shell_command(cmd, raise_on_failure: false) do |line| + QA::Runtime::Logger.debug(line.chomp) + break true if line.include?('the following nodes are not healthy') && line.include?(node) end end end def wait_for_health_check_all_nodes - wait_for_health_check(@primary_node) - wait_for_health_check(@secondary_node) - wait_for_health_check(@tertiary_node) + wait_for_gitaly_health_check(@primary_node) + wait_for_gitaly_health_check(@secondary_node) + wait_for_gitaly_health_check(@tertiary_node) end - def wait_for_health_check(node) + def wait_for_gitaly_health_check(node) QA::Runtime::Logger.info("Waiting for health check on #{node}") + Support::Waiter.wait_until(max_duration: 120, sleep_interval: 1, raise_on_failure: true) do + # gitaly runs a grpc server on port 8075, which will return an error 'Connection refused' until such time it is ready + wait_until_shell_command("docker exec #{@praefect} bash -c 'curl #{node}:8075'") do |line| + break if line.include?('curl: (1) Received HTTP/0.9 when not allowed') + + QA::Runtime::Logger.debug(line.chomp) + end + end wait_until_node_is_marked_as_healthy_storage(node) end def wait_for_primary_node_health_check - wait_for_health_check(@primary_node) + wait_for_gitaly_health_check(@primary_node) end def wait_for_secondary_node_health_check - wait_for_health_check(@secondary_node) + wait_for_gitaly_health_check(@secondary_node) end def wait_for_tertiary_node_health_check - wait_for_health_check(@tertiary_node) + wait_for_gitaly_health_check(@tertiary_node) end def wait_for_health_check_failure(node) @@ -311,7 +304,6 @@ module QA shell sql_to_docker_exec_cmd("SELECT count(*) FROM healthy_storages WHERE storage = '#{node}';") do |line| result << line end - QA::Runtime::Logger.debug("result is ---#{result}") result[2].to_i == 0 end end @@ -322,21 +314,10 @@ module QA shell sql_to_docker_exec_cmd("SELECT count(*) FROM healthy_storages WHERE storage = '#{node}';") do |line| result << line end - - QA::Runtime::Logger.debug("result is ---#{result}") result[2].to_i == 1 end end - def wait_for_gitaly_check - Support::Waiter.wait_until(max_duration: 120, sleep_interval: 1, raise_on_failure: true) do - wait_until_shell_command("docker exec #{@gitlab} bash -c 'gitlab-rake gitlab:git:fsck'") do |line| - QA::Runtime::Logger.debug(line.chomp) - line.include?('Done') - end - end - end - # Waits until there is an increase in the number of reads for # any node compared to the number of reads provided. If a node # has no pre-read data, consider it to have had zero reads. @@ -354,12 +335,6 @@ module QA data.find(-> {{ value: 0 }}) { |item| item[:node] == node }[:value] end - def wait_for_reliable_connection - QA::Runtime::Logger.info('Wait until GitLab and Praefect can communicate reliably') - wait_for_sql_ping - wait_for_storage_nodes - end - def wait_for_replication(project_id) Support::Waiter.wait_until(sleep_interval: 1) { replication_queue_incomplete_count == 0 && replicated?(project_id) } end @@ -414,10 +389,34 @@ module QA end def remove_tracked_praefect_repository(relative_path, virtual_storage) - cmd = "gitlab-ctl praefect remove-repository --repository-relative-path #{relative_path} --virtual-storage-name #{virtual_storage}" + cmd = "gitlab-ctl praefect remove-repository --repository-relative-path #{relative_path} --virtual-storage-name #{virtual_storage} --apply" shell "docker exec #{@praefect} bash -c '#{cmd}'" end + # set_replication_factor assigns or unassigns random storage nodes as necessary to reach the desired replication factor for a repository + def set_replication_factor(relative_path, virtual_storage, factor) + cmd = "/opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml set-replication-factor -repository #{relative_path} -virtual-storage #{virtual_storage} -replication-factor #{factor}" + shell "docker exec #{@praefect} bash -c '#{cmd}'" + end + + # get_replication_storages retrieves a list of currently assigned storages for a repository + def get_replication_storages(relative_path, virtual_storage) + storage_repositories = [] + query = "SELECT storage FROM repository_assignments WHERE relative_path='#{relative_path}' AND virtual_storage='#{virtual_storage}';" + shell(sql_to_docker_exec_cmd(query)) { |line| storage_repositories << line.strip } + # Returned data from query will be in format + # storage + # -------- + # gitaly1 + # gitaly3 + # gitaly2 + # (3 rows) + # + + # remove 2 header rows and last 2 rows from query response (including blank line) + storage_repositories[2..-3] + end + def add_repo_to_disk(node, repo_path) cmd = "GIT_DIR=. git init --initial-branch=main /var/opt/gitlab/git-data/repositories/#{repo_path}" shell "docker exec --user git #{node} bash -c '#{cmd}'" @@ -452,7 +451,7 @@ module QA end def repository_replicated_to_disk?(node, relative_path) - Support::Waiter.wait_until(max_duration: 300, sleep_interval: 3, raise_on_failure: false) do + Support::Waiter.wait_until(max_duration: 300, sleep_interval: 1, raise_on_failure: false) do result = [] shell sql_to_docker_exec_cmd("SELECT count(*) FROM storage_repositories where relative_path='#{relative_path}';") do |line| result << line diff --git a/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_group_spec.rb index 799a5f7eaf2..a6655471591 100644 --- a/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_group_spec.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true module QA - # run only base UI validation on staging because test requires top level group creation which is problematic - # on staging environment - RSpec.describe 'Manage', :requires_admin, except: { subdomain: :staging } do + RSpec.describe 'Manage', :requires_admin do describe 'Gitlab migration' do let(:import_wait_duration) { { max_duration: 300, sleep_interval: 2 } } let(:admin_api_client) { Runtime::API::Client.as_admin } @@ -22,9 +20,18 @@ module QA end end + let(:destination_group) do + Resource::Group.fabricate_via_api! do |group| + group.api_client = api_client + group.sandbox = sandbox + group.path = "destination-group-for-import-#{SecureRandom.hex(4)}" + end + end + let(:source_group) do - Resource::Sandbox.fabricate_via_api! do |group| + Resource::Group.fabricate_via_api! do |group| group.api_client = api_client + group.sandbox = sandbox group.path = "source-group-for-import-#{SecureRandom.hex(4)}" group.avatar = File.new('qa/fixtures/designs/tanuki.jpg', 'r') end @@ -33,8 +40,8 @@ module QA let(:imported_group) do Resource::BulkImportGroup.fabricate_via_api! do |group| group.api_client = api_client - group.sandbox = sandbox - group.source_group_path = source_group.path + group.sandbox = destination_group + group.source_group = source_group end end @@ -167,7 +174,6 @@ module QA expect { imported_group.import_status }.to eventually_eq('finished').within(import_wait_duration) imported_member = imported_group.reload!.members.find { |usr| usr.username == member.username } - aggregate_failures do expect(imported_member).not_to be_nil expect(imported_member.access_level).to eq(Resource::Members::AccessLevel::DEVELOPER) diff --git a/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_issue_spec.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_issue_spec.rb new file mode 100644 index 00000000000..8a2a382ac45 --- /dev/null +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_issue_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require_relative 'gitlab_project_migration_common' + +module QA + RSpec.describe 'Manage', :requires_admin do + describe 'Gitlab migration', quarantine: { + only: { job: 'praefect' }, + type: :investigating, + issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/348999' + } do + include_context 'with gitlab project migration' + + context 'with project issues' do + let!(:source_issue) do + Resource::Issue.fabricate_via_api! do |issue| + issue.api_client = api_client + issue.project = source_project + issue.labels = %w[label_one label_two] + end + end + + let!(:source_comment) { source_issue.add_comment(body: 'This is a test comment!') } + + let(:imported_issues) { imported_projects.first.issues } + + let(:imported_issue) do + issue = imported_issues.first + Resource::Issue.init do |resource| + resource.api_client = api_client + resource.project = imported_projects.first + resource.iid = issue[:iid] + end + end + + let(:imported_comments) { imported_issue.comments } + + it( + 'successfully imports issue', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347608' + ) do + expect_import_finished + + aggregate_failures do + expect(imported_issues.count).to eq(1) + expect(imported_issue).to eq(source_issue.reload!) + + expect(imported_comments.count).to eq(1) + expect(imported_comments.first[:body]).to include(source_comment[:body]) + end + end + end + end + end +end diff --git a/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_mr_spec.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_mr_spec.rb new file mode 100644 index 00000000000..9dce9bff3c1 --- /dev/null +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_mr_spec.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require_relative 'gitlab_project_migration_common' + +module QA + RSpec.describe 'Manage', :requires_admin do + describe 'Gitlab migration', quarantine: { + only: { job: 'praefect' }, + type: :investigating, + issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/348999' + } do + include_context 'with gitlab project migration' + + context 'with merge request' do + let!(:source_project_with_readme) { true } + + let!(:other_user) do + Resource::User + .fabricate_via_api! { |usr| usr.api_client = admin_api_client } + .tap do |usr| + usr.set_public_email + source_project.add_member(usr, Resource::Members::AccessLevel::MAINTAINER) + end + end + + let!(:source_mr) do + Resource::MergeRequest.fabricate_via_api! do |mr| + mr.project = source_project + mr.api_client = Runtime::API::Client.new(user: other_user) + end + end + + let!(:source_comment) { source_mr.add_comment('This is a test comment!') } + + let(:imported_mrs) { imported_project.merge_requests } + let(:imported_mr_comments) { imported_mr.comments } + + let(:imported_mr) do + Resource::MergeRequest.init do |mr| + mr.project = imported_project + mr.iid = imported_mrs.first[:iid] + mr.api_client = api_client + end + end + + after do + other_user.remove_via_api! + end + + it( + 'successfully imports merge request', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348478' + ) do + expect_import_finished + + aggregate_failures do + expect(imported_mrs.count).to eq(1) + # TODO: remove custom comparison after member migration is implemented + # https://gitlab.com/gitlab-org/gitlab/-/issues/341886 + expect(imported_mr.comparable.except(:author)).to eq(source_mr.reload!.comparable.except(:author)) + + expect(imported_mr_comments.count).to eq(1) + expect(imported_mr_comments.first[:body]).to include(source_comment[:body]) + # Comment will have mention of original user since members are not migrated yet + expect(imported_mr_comments.first[:body]).to include(other_user.name) + end + end + end + end + end +end diff --git a/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_project_spec.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_project_spec.rb new file mode 100644 index 00000000000..a0c758c99e6 --- /dev/null +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_project_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require_relative 'gitlab_project_migration_common' + +module QA + RSpec.describe 'Manage', :requires_admin do + describe 'Gitlab migration', quarantine: { + only: { job: 'praefect' }, + type: :investigating, + issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/348999' + } do + include_context 'with gitlab project migration' + + context 'with uninitialized project' do + it( + 'successfully imports project', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347610' + ) do + expect_import_finished + + expect(imported_projects.first).to eq(source_project) + end + end + + context 'with repository' do + let(:source_project_with_readme) { true } + let(:source_commits) { source_project.commits.map { |c| c.except(:web_url) } } + let(:source_tags) do + source_project.repository_tags.tap do |tags| + tags.each { |t| t[:commit].delete(:web_url) } + end + end + + let(:source_branches) do + source_project.repository_branches.tap do |branches| + branches.each do |b| + b.delete(:web_url) + b[:commit].delete(:web_url) + end + end + end + + let(:imported_commits) { imported_project.commits.map { |c| c.except(:web_url) } } + let(:imported_tags) do + imported_project.repository_tags.tap do |tags| + tags.each { |t| t[:commit].delete(:web_url) } + end + end + + let(:imported_branches) do + imported_project.repository_branches.tap do |branches| + branches.each do |b| + b.delete(:web_url) + b[:commit].delete(:web_url) + end + end + end + + before do + source_project.create_repository_branch('test-branch') + source_project.create_repository_tag('v0.0.1') + end + + it( + 'successfully imports repository', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347570' + ) do + expect_import_finished + + aggregate_failures do + expect(imported_commits).to match_array(source_commits) + expect(imported_tags).to match_array(source_tags) + expect(imported_branches).to match_array(source_branches) + end + end + end + + context 'with wiki' do + before do + source_project.create_wiki_page(title: 'Import test project wiki', content: 'Wiki content') + end + + it( + 'successfully imports project wiki', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347567' + ) do + expect_import_finished + + expect(imported_projects.first.wikis).to eq(source_project.wikis) + end + end + end + end +end diff --git a/qa/qa/specs/features/api/1_manage/migration/gitlab_project_migration_common.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_project_migration_common.rb new file mode 100644 index 00000000000..827ebc1f5e2 --- /dev/null +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_project_migration_common.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module QA + RSpec.shared_context 'with gitlab project migration' do + let(:source_project_with_readme) { false } + let(:import_wait_duration) { { max_duration: 300, sleep_interval: 2 } } + let(:admin_api_client) { Runtime::API::Client.as_admin } + let(:user) do + Resource::User.fabricate_via_api! do |usr| + usr.api_client = admin_api_client + usr.hard_delete_on_api_removal = true + end + end + + let(:api_client) { Runtime::API::Client.new(user: user) } + + let(:sandbox) do + Resource::Sandbox.fabricate_via_api! do |group| + group.api_client = admin_api_client + end + end + + let(:destination_group) do + Resource::Group.fabricate_via_api! do |group| + group.api_client = api_client + group.sandbox = sandbox + group.path = "destination-group-for-import-#{SecureRandom.hex(4)}" + end + end + + let(:source_group) do + Resource::Group.fabricate_via_api! do |group| + group.api_client = api_client + group.path = "source-group-for-import-#{SecureRandom.hex(4)}" + end + end + + let(:source_project) do + Resource::Project.fabricate_via_api! do |project| + project.api_client = api_client + project.group = source_group + project.initialize_with_readme = source_project_with_readme + end + end + + let(:imported_group) do + Resource::BulkImportGroup.fabricate_via_api! do |group| + group.api_client = api_client + group.sandbox = destination_group + group.source_group = source_group + end + end + + let(:imported_projects) { imported_group.reload!.projects } + let(:imported_project) { imported_projects.first } + + let(:import_failures) do + imported_group.import_details.sum([]) { |details| details[:failures] } + end + + def expect_import_finished + imported_group # trigger import + + expect { imported_group.import_status }.to eventually_eq('finished').within(import_wait_duration) + expect(imported_projects.count).to eq(1), 'Expected to have 1 imported project' + end + + before do + Runtime::Feature.enable(:bulk_import_projects) + + sandbox.add_member(user, Resource::Members::AccessLevel::MAINTAINER) + source_project # fabricate source group and project + end + + after do |example| + # Checking for failures in the test currently makes test very flaky + # Just log in case of failure until cause of network errors is found + Runtime::Logger.warn("Import failures: #{import_failures}") if example.exception && !import_failures.empty? + + user.remove_via_api! + ensure + Runtime::Feature.disable(:bulk_import_projects) + end + end +end diff --git a/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb b/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb index e47e5d22e5e..6a31d173440 100644 --- a/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb +++ b/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb @@ -37,7 +37,7 @@ module QA push.file_name = 'test.txt' push.file_content = "# This is a test project named #{@project.name}" push.commit_message = 'Add test.txt' - push.branch_name = 'new_branch' + push.branch_name = "new_branch_#{SecureRandom.hex(8)}" push.user = @user end end.to raise_error(QA::Support::Run::CommandError, /You are not allowed to push code to this project/) @@ -48,7 +48,7 @@ module QA Resource::File.fabricate_via_api! do |file| file.api_client = @user_api_client file.project = @project - file.branch = 'new_branch' + file.branch = "new_branch_#{SecureRandom.hex(8)}" file.commit_message = 'Add new file' file.name = 'test.txt' file.content = "New file" @@ -61,7 +61,7 @@ module QA Resource::Repository::Commit.fabricate_via_api! do |commit| commit.api_client = @user_api_client commit.project = @project - commit.branch = 'new_branch' + commit.branch = "new_branch_#{SecureRandom.hex(8)}" commit.start_branch = @project.default_branch commit.commit_message = 'Add new file' commit.add_files([ diff --git a/qa/qa/specs/features/api/3_create/gitaly/automatic_failover_and_recovery_spec.rb b/qa/qa/specs/features/api/3_create/gitaly/automatic_failover_and_recovery_spec.rb index 51927a30987..6a9be19efdd 100644 --- a/qa/qa/specs/features/api/3_create/gitaly/automatic_failover_and_recovery_spec.rb +++ b/qa/qa/specs/features/api/3_create/gitaly/automatic_failover_and_recovery_spec.rb @@ -49,7 +49,6 @@ module QA # for Gitaly to be ready for writes again praefect_manager.stop_primary_node praefect_manager.wait_for_primary_node_health_check_failure - praefect_manager.wait_for_gitaly_check Resource::Repository::Commit.fabricate_via_api! do |commit| commit.project = project diff --git a/qa/qa/specs/features/api/3_create/gitaly/praefect_connectivity_spec.rb b/qa/qa/specs/features/api/3_create/gitaly/praefect_connectivity_spec.rb new file mode 100644 index 00000000000..28469b99d04 --- /dev/null +++ b/qa/qa/specs/features/api/3_create/gitaly/praefect_connectivity_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Create' do + context 'Praefect connectivity commands', :orchestrated, :gitaly_cluster do + praefect_manager = Service::PraefectManager.new + + before do + praefect_manager.start_all_nodes + end + + context 'in a healthy environment' do + it 'confirms healthy connection to database', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349937' do + expect(praefect_manager.praefect_sql_ping_healthy?).to be true + end + + it 'confirms healthy connection to gitaly nodes', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349938' do + expect(praefect_manager.wait_for_dial_nodes_successful).to be true + end + end + + context 'in an unhealthy environment' do + it 'diagnoses unhealthy connection to database', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349939' do + praefect_manager.stop_node(praefect_manager.postgres) + expect(praefect_manager.praefect_sql_ping_healthy?).to be false + end + + it 'diagnoses connection issues to gitaly nodes', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349940' do + praefect_manager.stop_node(praefect_manager.primary_node) + praefect_manager.stop_node(praefect_manager.tertiary_node) + expect(praefect_manager.praefect_dial_nodes_status?(praefect_manager.primary_node, false)).to be true + expect(praefect_manager.praefect_dial_nodes_status?(praefect_manager.secondary_node)).to be true + expect(praefect_manager.praefect_dial_nodes_status?(praefect_manager.tertiary_node, false)).to be true + + praefect_manager.stop_node(praefect_manager.secondary_node) + expect(praefect_manager.praefect_dial_nodes_status?(praefect_manager.secondary_node, false)).to be true + end + end + end + end +end diff --git a/qa/qa/specs/features/api/3_create/gitaly/praefect_repo_sync_spec.rb b/qa/qa/specs/features/api/3_create/gitaly/praefect_repo_sync_spec.rb index cc49e408954..e27f37abedf 100644 --- a/qa/qa/specs/features/api/3_create/gitaly/praefect_repo_sync_spec.rb +++ b/qa/qa/specs/features/api/3_create/gitaly/praefect_repo_sync_spec.rb @@ -2,7 +2,7 @@ module QA RSpec.describe 'Create' do - context 'Praefect repository commands', :orchestrated, :gitaly_cluster, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/347415', type: :investigating } do + context 'Praefect repository commands', :orchestrated, :gitaly_cluster do let(:praefect_manager) { Service::PraefectManager.new } let(:repo1) { { "relative_path" => "@hashed/repo1.git", "storage" => "gitaly1", "virtual_storage" => "default" } } @@ -59,6 +59,18 @@ module QA untracked_repositories = praefect_manager.list_untracked_repositories expect(untracked_repositories).not_to include(repo1) end + + it 'allows admin to control the number of replicas of data', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347566' do + praefect_manager.track_repository_in_praefect(repo1['relative_path'], repo1['storage'], repo1['virtual_storage']) + + praefect_manager.set_replication_factor(repo1['relative_path'], repo1['virtual_storage'], 2) + replication_storages = praefect_manager.get_replication_storages(repo1['relative_path'], repo1['virtual_storage']) + expect(replication_storages).to have_attributes(size: 2) + + praefect_manager.set_replication_factor(repo1['relative_path'], repo1['virtual_storage'], 3) + replication_storages = praefect_manager.get_replication_storages(repo1['relative_path'], repo1['virtual_storage']) + expect(replication_storages).to eq(%w(gitaly1 gitaly2 gitaly3)) + end end end end diff --git a/qa/qa/specs/features/browser_ui/1_manage/group/bulk_import_group_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/group/gitlab_migration_group_spec.rb index 74125b092b8..a18e22f52f1 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/group/bulk_import_group_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/group/gitlab_migration_group_spec.rb @@ -1,9 +1,8 @@ # frozen_string_literal: true module QA - RSpec.describe 'Manage', :requires_admin do - describe 'Bulk group import' do - let!(:staging?) { Runtime::Scenario.gitlab_address.include?('staging.gitlab.com') } + describe 'Manage', :requires_admin do + describe 'Gitlab migration' do let!(:admin_api_client) { Runtime::API::Client.as_admin } let!(:user) do Resource::User.fabricate_via_api! do |usr| @@ -32,7 +31,7 @@ module QA Resource::BulkImportGroup.init do |group| group.api_client = api_client group.sandbox = sandbox - group.source_group_path = source_group.path + group.source_group = source_group end end diff --git a/qa/qa/specs/features/browser_ui/1_manage/login/register_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/login/register_spec.rb index 16f8df5a90d..098c0b3ba63 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/login/register_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/login/register_spec.rb @@ -64,6 +64,7 @@ module QA Page::Profile::Accounts::Show.perform do |show| show.delete_account(user.password) end + Support::Waiter.wait_until { !user.exists? } end it 'allows recreating with same credentials', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347868' do @@ -83,7 +84,7 @@ module QA end after do - @recreated_user.remove_via_api! + @recreated_user&.remove_via_api! end def admin_api_client @@ -117,11 +118,12 @@ module QA Flow::Login.sign_in(as: @user, skip_page_validation: true) - Page::Registration::Welcome.perform(&:click_get_started_button_if_available) + Flow::UserOnboarding.onboard_user - Page::Main::Menu.perform do |menu| - expect(menu).to have_personal_area - end + # In development env and .com the user is asked to create a group and a project which can be skipped for + # the purpose of this test + Runtime::Browser.visit(:gitlab, Page::Dashboard::Welcome) + Page::Main::Menu.perform(&:has_personal_area?) end after do diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb index 6d09c8b1316..895027a588d 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb @@ -1,7 +1,11 @@ # frozen_string_literal: true module QA - RSpec.describe 'Manage', :requires_admin do + RSpec.describe 'Manage', :requires_admin, quarantine: { + issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/350598', + type: :needs_update, + only: { subdomain: :staging } + } do describe 'Add project member' do before do Runtime::Feature.enable(:invite_members_group_modal) diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb index 7f40818da03..0063ce2613a 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb @@ -2,7 +2,7 @@ module QA RSpec.describe 'Manage', :smoke do - describe 'Project', :requires_admin do + describe 'Project' do shared_examples 'successful project creation' do it 'creates a new project' do Page::Project::Show.perform do |project_page| @@ -17,7 +17,6 @@ module QA end before do - Runtime::Feature.enable(:paginatable_namespace_drop_down_for_project_creation) Flow::Login.sign_in project end diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/invite_group_to_project_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/invite_group_to_project_spec.rb new file mode 100644 index 00000000000..6997447411a --- /dev/null +++ b/qa/qa/specs/features/browser_ui/1_manage/project/invite_group_to_project_spec.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module QA + # Tagging with issue for a transient invite group modal search bug, but does not require quarantine at this time + RSpec.describe 'Manage', :requires_admin, :transient, issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/349379' do + describe 'Invite group' do + shared_examples 'invites group to project' do + it 'verifies group is added and members can access project with correct access level' do + Page::Project::Menu.perform(&:click_members) + Page::Project::Members.perform do |project_members| + project_members.invite_group(group.path, 'Developer') + + expect(project_members).to have_group(group.path) + end + + Flow::Login.sign_in(as: @user) + + Page::Dashboard::Projects.perform do |projects| + expect(projects).to have_project_with_access_role(project.name, 'Developer') + end + + project.visit! + + Page::Project::Show.perform do |project_page| + expect(project_page).to have_name(project.name) + end + end + end + + before(:context) do + Runtime::Feature.enable(:invite_members_group_modal) + @user = Resource::User.fabricate_or_use(Runtime::Env.gitlab_qa_username_1, Runtime::Env.gitlab_qa_password_1) + end + + before do + Flow::Login.sign_in + group.add_member(@user, Resource::Members::AccessLevel::MAINTAINER) + project.visit! + end + + context 'to personal namespace project', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349223' do + let(:group) do + Resource::Group.fabricate_via_api! do |group| + group.path = "group-for-personal-project-#{SecureRandom.hex(8)}" + end + end + + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'personal-namespace-project' + project.personal_namespace = Runtime::User.username + project.visibility = :private + project.description = 'test personal namespace project' + end + end + + it_behaves_like 'invites group to project' + end + + context 'to group project', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349340' do + let(:group) do + Resource::Group.fabricate_via_api! do |group| + group.path = "group-for-group-project-#{SecureRandom.hex(8)}" + end + end + + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'group-project' + project.visibility = :private + project.description = 'test group project' + end + end + + it_behaves_like 'invites group to project' + end + + after do + project&.remove_via_api! + group&.remove_via_api! + end + + after(:context) do + Runtime::Feature.disable(:invite_members_group_modal) + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/project_access_token_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/project_access_token_spec.rb index be8567ee0b6..c2bd61155b1 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/project/project_access_token_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/project/project_access_token_spec.rb @@ -9,7 +9,7 @@ module QA expect(project_access_token.token).not_to be_nil project_access_token.revoke_via_ui! - expect(page).to have_text("Revoked project access token #{project_access_token.name}!") + expect(page).to have_text("Revoked access token #{project_access_token.name}!") end after do diff --git a/qa/qa/specs/features/browser_ui/1_manage/user/follow_user_activity_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/user/follow_user_activity_spec.rb index 43100929acd..87b51edef08 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/user/follow_user_activity_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/user/follow_user_activity_spec.rb @@ -19,7 +19,7 @@ module QA group = QA::Resource::Group.fabricate_via_api! do |group| group.path = "group_for_follow_user_activity_#{SecureRandom.hex(8)}" end - group.add_member(user) + group.add_member(user, Resource::Members::AccessLevel::MAINTAINER) group end diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb index d198d79c5fe..b0c6d01e8ca 100644 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb @@ -2,7 +2,11 @@ module QA RSpec.describe 'Create' do - describe 'Merge request creation from fork' do + describe 'Merge request creation from fork', quarantine: { + only: { subdomain: %i[canary production] }, + issue: "https://gitlab.com/gitlab-org/gitlab/-/issues/343801", + type: :investigation + } do let(:merge_request) do Resource::MergeRequestFromFork.fabricate_via_browser_ui! do |merge_request| merge_request.fork_branch = 'feature-branch' diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/view_merge_request_merge_ref_diff_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/view_merge_request_merge_ref_diff_spec.rb deleted file mode 100644 index 0785b32b225..00000000000 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/view_merge_request_merge_ref_diff_spec.rb +++ /dev/null @@ -1,90 +0,0 @@ -# frozen_string_literal: true - -module QA - RSpec.describe 'Create', :requires_admin, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/261793', type: :investigating } do - describe 'View merge request merge-ref diff' do - let(:project) do - Resource::Project.fabricate_via_api! do |project| - project.name = 'merge-ref-diff' - end - end - - let(:merge_request) do - Resource::MergeRequest.fabricate_via_api! do |merge_request| - merge_request.project = project - merge_request.title = 'This is a merge request' - merge_request.description = '... for viewing merge-ref and merge-base diffs' - merge_request.file_content = 'This exists on the source branch only' - end - end - - let(:new_file_name) { "added_file-#{SecureRandom.hex(8)}.txt" } - - context 'when the feature flag default_merge_ref_for_diffs is enabled' do - before do - Runtime::Feature.enable('default_merge_ref_for_diffs', project: project) - - commit_to_branch(merge_request.target_branch, new_file_name) - commit_to_branch(merge_request.source_branch, new_file_name) - - Flow::Login.sign_in - - merge_request.visit! - end - - it 'views the merge-ref diff by default', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347651' do - Page::MergeRequest::Show.perform do |mr_page| - mr_page.click_diffs_tab - mr_page.click_target_version_dropdown - - expect(mr_page.version_dropdown_content).to include("#{project.default_branch} (HEAD)") - expect(mr_page.version_dropdown_content).not_to include("#{project.default_branch} (base)") - expect(mr_page).to have_file(merge_request.file_name) - expect(mr_page).not_to have_file(new_file_name) - end - end - end - - context 'when the feature flag default_merge_ref_for_diffs is disabled' do - before do - Runtime::Feature.disable('default_merge_ref_for_diffs', project: project) - - commit_to_branch(merge_request.target_branch, new_file_name) - commit_to_branch(merge_request.source_branch, new_file_name) - - Flow::Login.sign_in - - merge_request.visit! - end - - it 'views the merge-base diff by default', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347650' do - Page::MergeRequest::Show.perform do |mr_page| - mr_page.click_diffs_tab - mr_page.click_target_version_dropdown - - expect(mr_page.version_dropdown_content).to include("#{project.default_branch} (HEAD)") - expect(mr_page.version_dropdown_content).to include("#{project.default_branch} (base)") - expect(mr_page).to have_file(merge_request.file_name) - expect(mr_page).to have_file(new_file_name) - end - end - end - - def commit_to_branch(branch, file) - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.project = merge_request.project - commit.branch = branch - commit.commit_message = "Add new file on #{branch}" - commit.add_files( - [ - { - file_path: file, - content: "This exists on source and target branches" - } - ] - ) - end - end - end - end -end diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb index a98925eab98..0bd470fcb77 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb @@ -24,8 +24,6 @@ module QA proj.initialize_with_readme = true end - Runtime::Feature.enable(:delete_branch_confirmation_modals, project: project) - master_branch = project.default_branch Git::Repository.perform do |repository| diff --git a/qa/qa/specs/features/browser_ui/3_create/wiki/content_editor_spec.rb b/qa/qa/specs/features/browser_ui/3_create/wiki/content_editor_spec.rb index a063acbe146..67eee66b3d6 100644 --- a/qa/qa/specs/features/browser_ui/3_create/wiki/content_editor_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/wiki/content_editor_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Create', :requires_admin do # remove :requires_admin once the ff is enabled by default in https://gitlab.com/gitlab-org/gitlab/-/issues/345398 + RSpec.describe 'Create', :requires_admin, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/350220', type: :investigating } do # remove :requires_admin once the ff is enabled by default in https://gitlab.com/gitlab-org/gitlab/-/issues/345398 context 'Content Editor' do let(:initial_wiki) { Resource::Wiki::ProjectPage.fabricate_via_api! } let(:page_title) { 'Content Editor Page' } diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_lint_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_lint_spec.rb new file mode 100644 index 00000000000..8f3284662d7 --- /dev/null +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_lint_spec.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Verify' do + describe 'Pipeline editor' do + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'pipeline-editor-project' + end + end + + let!(:commit) do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files( + [ + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + stages: + - stage1 + - stage2 + + job1: + stage: stage1 + script: echo 'Done.' + + job2: + stage: stage2 + script: echo 'Done.' + YAML + } + ] + ) + end + end + + before do + Flow::Login.sign_in + project.visit! + Page::Project::Menu.perform(&:go_to_pipeline_editor) + end + + after do + project&.remove_via_api! + end + + context 'when CI has valid syntax' do + it 'shows valid validations', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349128' do + Page::Project::PipelineEditor::Show.perform do |show| + aggregate_failures do + expect(show.ci_syntax_validate_message).to have_content('CI configuration is valid') + + show.go_to_visualize_tab + { stage1: 'job1', stage2: 'job2' }.each_pair do |stage, job| + expect(show).to have_stage(stage), "Pipeline graph does not have stage #{stage}." + expect(show).to have_job(job), "Pipeline graph does not have job #{job}." + end + + show.go_to_lint_tab + expect(show.tab_alert_message).to have_content('Syntax is correct') + + show.go_to_view_merged_yaml_tab + expect(show).to have_source_editor + end + end + end + end + + context 'when CI has invalid syntax' do + it 'shows invalid validations', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349129' do + invalid_msg = 'syntax is invalid' + + Page::Project::PipelineEditor::Show.perform do |show| + show.write_to_editor(SecureRandom.hex(10)) + + aggregate_failures do + show.go_to_visualize_tab + expect(show.tab_alert_message).to have_content(invalid_msg) + + show.go_to_lint_tab + expect(show.tab_alert_message).to have_content('Syntax is incorrect') + + show.go_to_view_merged_yaml_tab + expect(show.tab_alert_message).to have_content(invalid_msg) + + expect(show.ci_syntax_validate_message).to have_content('CI configuration is invalid') + end + end + end + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/update_ci_file_with_pipeline_editor_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/update_ci_file_with_pipeline_editor_spec.rb new file mode 100644 index 00000000000..00c5d4c74d4 --- /dev/null +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/update_ci_file_with_pipeline_editor_spec.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Verify' do + describe 'Update CI file with pipeline editor' do + let(:random_test_string) { SecureRandom.hex(10) } + + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'pipeline-editor-project' + end + end + + let!(:runner) do + Resource::Runner.fabricate_via_api! do |runner| + runner.project = project + runner.name = random_test_string + runner.tags = [random_test_string] + end + end + + let!(:commit) do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files( + [ + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + test_job: + tags: ['#{random_test_string}'] + script: + - echo "Simple test!" + YAML + } + ] + ) + end + end + + before do + Flow::Login.sign_in + project.visit! + Support::Waiter.wait_until { !project.pipelines.empty? && project.pipelines.first[:status] == 'success' } + Page::Project::Menu.perform(&:go_to_pipeline_editor) + end + + after do + [runner, project].each(&:remove_via_api!) + end + + it 'creates new pipeline and target branch', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/349005' do + Page::Project::PipelineEditor::Show.perform do |show| + show.write_to_editor(random_test_string) + show.set_target_branch(random_test_string) + show.submit_changes + + Support::Waiter.wait_until { project.pipelines.size > 1 } + + aggregate_failures do + expect(show.target_branch_name).to eq(random_test_string) + expect(show.current_branch).to eq(random_test_string) + expect(show.editing_content).to have_content(random_test_string) + expect { show.pipeline_id }.to eventually_eq(project.pipelines.pluck(:id).max).within(max_duration: 60, sleep_interval: 3) + end + end + + expect(project).to have_branch(random_test_string) + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/5_package/container_registry/online_garbage_collection_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry/online_garbage_collection_spec.rb index e8d936e67b1..56e3ec82388 100644 --- a/qa/qa/specs/features/browser_ui/5_package/container_registry/online_garbage_collection_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/container_registry/online_garbage_collection_spec.rb @@ -2,8 +2,7 @@ module QA RSpec.describe 'Package' do - # TODO: Remove :requires_admin when the `Runtime::Feature.enable` method call is removed - describe 'Container Registry Online Garbage Collection', :registry_gc, :requires_admin, only: { subdomain: %i[pre] } do + describe 'Container Registry Online Garbage Collection', :registry_gc, only: { subdomain: %i[pre] } do let(:group) { Resource::Group.fabricate_via_api! } let(:imported_project) do @@ -65,8 +64,6 @@ module QA end before do - Runtime::Feature.enable(:paginatable_namespace_drop_down_for_project_creation) - Flow::Login.sign_in imported_project diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb index c58cdec622d..70b31c1beca 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Package Registry', :orchestrated, :packages, :reliable, :object_storage do + RSpec.describe 'Package Registry', :orchestrated, :packages, :object_storage do describe 'npm instance level endpoint' do using RSpec::Parameterized::TableSyntax include Runtime::Fixtures diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb index cec902e073a..e25a742493b 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Package Registry', :orchestrated, :packages, :reliable, :object_storage do + RSpec.describe 'Package Registry', :orchestrated, :packages, :object_storage do describe 'npm project level endpoint' do using RSpec::Parameterized::TableSyntax include Runtime::Fixtures diff --git a/qa/qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_independent_relationship_spec.rb b/qa/qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_independent_relationship_spec.rb index 74a81ff429d..ef0c8d35c37 100644 --- a/qa/qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_independent_relationship_spec.rb +++ b/qa/qa/specs/features/browser_ui/6_release/pipeline/parent_child_pipelines_independent_relationship_spec.rb @@ -13,7 +13,7 @@ module QA Resource::Runner.fabricate_via_api! do |runner| runner.project = project runner.name = project.name - runner.tags = ["#{project.name}"] + runner.tags = [project.name] end end diff --git a/qa/qa/specs/features/browser_ui/8_monitor/.gitkeep b/qa/qa/specs/features/browser_ui/8_monitor/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/qa/qa/specs/features/browser_ui/8_monitor/.gitkeep diff --git a/qa/qa/specs/features/browser_ui/8_monitor/all_monitor_core_features_spec.rb b/qa/qa/specs/features/browser_ui/8_monitor/all_monitor_core_features_spec.rb deleted file mode 100644 index c13d2d2dddf..00000000000 --- a/qa/qa/specs/features/browser_ui/8_monitor/all_monitor_core_features_spec.rb +++ /dev/null @@ -1,142 +0,0 @@ -# frozen_string_literal: true -require_relative 'cluster_with_prometheus' - -module QA - RSpec.describe 'Monitor', :orchestrated, :kubernetes, :requires_admin, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/241448', type: :investigating } do - include_context "cluster with Prometheus installed" - - before do - Flow::Login.sign_in_unless_signed_in - @project.visit! - end - - it 'configures custom metrics', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348082' do - verify_add_custom_metric - verify_edit_custom_metric - verify_delete_custom_metric - end - - it 'duplicates to create dashboard to custom', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348070' do - Page::Project::Menu.perform(&:go_to_monitor_metrics) - - Page::Project::Monitor::Metrics::Show.perform do |on_dashboard| - on_dashboard.duplicate_dashboard - - expect(on_dashboard).to have_metrics - expect(on_dashboard).to have_edit_dashboard_enabled - end - end - - it 'verifies data on filtered deployed environment', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348071' do - Page::Project::Menu.perform(&:go_to_monitor_metrics) - - Page::Project::Monitor::Metrics::Show.perform do |on_dashboard| - on_dashboard.filter_environment - - expect(on_dashboard).to have_metrics - end - end - - it 'filters using the quick range', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348083' do - Page::Project::Menu.perform(&:go_to_monitor_metrics) - - Page::Project::Monitor::Metrics::Show.perform do |on_dashboard| - on_dashboard.show_last('30 minutes') - expect(on_dashboard).to have_metrics - - on_dashboard.show_last('3 hours') - expect(on_dashboard).to have_metrics - - on_dashboard.show_last('1 day') - expect(on_dashboard).to have_metrics - end - end - - it 'observes cluster health graph', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348074' do - Page::Project::Menu.perform(&:go_to_infrastructure_kubernetes) - - Page::Project::Infrastructure::Kubernetes::Index.perform do |cluster_list| - cluster_list.click_on_cluster(@cluster) - end - - Page::Project::Infrastructure::Kubernetes::Show.perform do |cluster_panel| - cluster_panel.open_health - cluster_panel.wait_for_cluster_health - end - end - - it 'uses templating variables for metrics dashboards', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347636' do - templating_dashboard_yml = Pathname - .new(__dir__) - .join('../../../../fixtures/metrics_dashboards/templating.yml') - - Resource::Repository::ProjectPush.fabricate! do |push| - push.project = @project - push.file_name = '.gitlab/dashboards/templating.yml' - push.file_content = File.read(templating_dashboard_yml) - push.commit_message = 'Add templating in dashboard file' - push.new_branch = false - end - - Page::Project::Menu.perform(&:go_to_monitor_metrics) - - Page::Project::Monitor::Metrics::Show.perform do |dashboard| - dashboard.select_dashboard('templating.yml') - - expect(dashboard).to have_template_metric('CPU usage GitLab Runner') - expect(dashboard).to have_template_metric('Memory usage Postgresql') - expect(dashboard).to have_templating_variable('GitLab Runner') - expect(dashboard).to have_templating_variable('Postgresql') - end - end - - private - - def verify_add_custom_metric - Page::Project::Menu.perform(&:go_to_integrations_settings) - Page::Project::Settings::Integrations.perform(&:click_on_prometheus_integration) - - Page::Project::Settings::Services::Prometheus.perform do |metrics_panel| - metrics_panel.click_on_new_metric - metrics_panel.add_custom_metric - end - - Page::Project::Menu.perform(&:go_to_monitor_metrics) - - Page::Project::Monitor::Metrics::Show.perform do |on_dashboard| - expect(on_dashboard).to have_custom_metric('HTTP Requests Total') - end - end - - def verify_edit_custom_metric - Page::Project::Menu.perform(&:go_to_integrations_settings) - Page::Project::Settings::Integrations.perform(&:click_on_prometheus_integration) - Page::Project::Settings::Services::Prometheus.perform do |metrics_panel| - metrics_panel.click_on_custom_metric('Business / HTTP Requests Total (req/sec)') - metrics_panel.edit_custom_metric - end - - Page::Project::Menu.perform(&:go_to_monitor_metrics) - - Page::Project::Monitor::Metrics::Show.perform do |on_dashboard| - expect(on_dashboard).to have_custom_metric('Throughput') - end - end - - def verify_delete_custom_metric - Page::Project::Menu.perform(&:go_to_integrations_settings) - Page::Project::Settings::Integrations.perform(&:click_on_prometheus_integration) - - Page::Project::Settings::Services::Prometheus.perform do |metrics_panel| - metrics_panel.click_on_custom_metric('Business / Throughput (req/sec)') - metrics_panel.delete_custom_metric - end - - Page::Project::Menu.perform(&:go_to_monitor_metrics) - - Page::Project::Monitor::Metrics::Show.perform do |on_dashboard| - expect(on_dashboard).not_to have_custom_metric('Throughput') - end - end - end -end diff --git a/qa/qa/specs/features/browser_ui/8_monitor/cluster_with_prometheus.rb b/qa/qa/specs/features/browser_ui/8_monitor/cluster_with_prometheus.rb deleted file mode 100644 index 19e49400d5e..00000000000 --- a/qa/qa/specs/features/browser_ui/8_monitor/cluster_with_prometheus.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true - -module QA - RSpec.shared_context "cluster with Prometheus installed" do - before :all do - @cluster = Service::KubernetesCluster.new(provider_class: Service::ClusterProvider::K3s).create! - @project = Resource::Project.fabricate_via_api! do |project| - project.name = 'monitoring-project' - project.auto_devops_enabled = true - project.template_name = 'express' - end - - deploy_project_with_prometheus - end - - def deploy_project_with_prometheus - %w[ - CODE_QUALITY_DISABLED TEST_DISABLED LICENSE_MANAGEMENT_DISABLED - SAST_DISABLED DAST_DISABLED DEPENDENCY_SCANNING_DISABLED - CONTAINER_SCANNING_DISABLED BROWSER_PERFORMANCE_DISABLED SECRET_DETECTION_DISABLED - ].each do |key| - Resource::CiVariable.fabricate_via_api! do |resource| - resource.project = @project - resource.key = key - resource.value = '1' - resource.masked = false - end - end - - Flow::Login.sign_in - - Resource::KubernetesCluster::ProjectCluster.fabricate! do |cluster_settings| - cluster_settings.project = @project - cluster_settings.cluster = @cluster - cluster_settings.install_runner = true - cluster_settings.install_ingress = true - cluster_settings.install_prometheus = true - end - - Resource::Pipeline.fabricate_via_api! do |pipeline| - pipeline.project = @project - end.visit! - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('build') - end - Page::Project::Job::Show.perform do |job| - expect(job).to be_successful(timeout: 600) - - job.click_element(:pipeline_path) - end - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('production') - end - Page::Project::Job::Show.perform do |job| - expect(job).to be_successful(timeout: 1200) - - job.click_element(:pipeline_path) - end - end - - after :all do - @cluster&.remove! - end - end -end diff --git a/qa/qa/specs/helpers/quarantine.rb b/qa/qa/specs/helpers/quarantine.rb index 49d91fc87cd..738c99efb28 100644 --- a/qa/qa/specs/helpers/quarantine.rb +++ b/qa/qa/specs/helpers/quarantine.rb @@ -11,9 +11,9 @@ module QA extend self # Skip tests in quarantine unless we explicitly focus on them. - def skip_or_run_quarantined_tests_or_contexts(filters, example) + def skip_or_run_quarantined_tests_or_contexts(example) if filters.key?(:quarantine) - included_filters = filters_other_than_quarantine(filters) + included_filters = filters_other_than_quarantine # If :quarantine is focused, skip the test/context unless its metadata # includes quarantine and any other filters @@ -29,18 +29,17 @@ module QA elsif example.metadata.key?(:quarantine) quarantine_tag = example.metadata[:quarantine] - if quarantine_tag.is_a?(Hash) && quarantine_tag&.key?(:only) && !ContextSelector.context_matches?(quarantine_tag[:only]) - # If the :quarantine hash contains :only, we respect that. - # For instance `quarantine: { only: { subdomain: :staging } }` will only quarantine the test when it runs against staging. - return - end + # If the :quarantine hash contains :only, we respect that. + # For instance `quarantine: { only: { subdomain: :staging } }` + # will only quarantine the test when it runs against staging. + return if quarantined_different_context?(quarantine_tag) example.metadata[:skip] = quarantine_message(quarantine_tag) end end - def filters_other_than_quarantine(filter) - filter.reject { |key, _| key == :quarantine } + def filters_other_than_quarantine + filters.reject { |key, _| key == :quarantine } end def quarantine_message(quarantine_tag) @@ -70,6 +69,14 @@ module QA (metadata.keys & included_filters.keys).empty? end + + def quarantined_different_context?(quarantine) + quarantine.is_a?(Hash) && quarantine.key?(:only) && !ContextSelector.context_matches?(quarantine[:only]) + end + + def filters + @filters ||= ::RSpec.configuration.inclusion_filter.rules + end end end end diff --git a/qa/qa/specs/runner.rb b/qa/qa/specs/runner.rb index d7d64834e7a..2c9e302fc56 100644 --- a/qa/qa/specs/runner.rb +++ b/qa/qa/specs/runner.rb @@ -9,6 +9,7 @@ module QA attr_accessor :tty, :tags, :options DEFAULT_TEST_PATH_ARGS = ['--', File.expand_path('./features', __dir__)].freeze + DEFAULT_STD_ARGS = [$stderr, $stdout].freeze def initialize @tty = false @@ -19,13 +20,11 @@ module QA def paths_from_knapsack allocator = Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator - QA::Runtime::Logger.info '' + QA::Runtime::Logger.info '==== Knapsack specs to execute =====' QA::Runtime::Logger.info 'Report specs:' QA::Runtime::Logger.info allocator.report_node_tests.join(', ') - QA::Runtime::Logger.info '' QA::Runtime::Logger.info 'Leftover specs:' QA::Runtime::Logger.info allocator.leftover_node_tests.join(', ') - QA::Runtime::Logger.info '' ['--', allocator.node_tests] end @@ -70,8 +69,15 @@ module QA ParallelRunner.run(args.flatten) elsif Runtime::Scenario.attributes[:loop] LoopRunner.run(args.flatten) + elsif Runtime::Scenario.attributes[:count_examples_only] + args.unshift('--dry-run') + out = StringIO.new + RSpec::Core::Runner.run(args.flatten, $stderr, out).tap do |status| + abort if status.nonzero? + end + $stdout.puts out.string.match(/(\d+) examples,/)[1] else - RSpec::Core::Runner.run(args.flatten, $stderr, $stdout).tap do |status| + RSpec::Core::Runner.run(args.flatten, *DEFAULT_STD_ARGS).tap do |status| abort if status.nonzero? end end diff --git a/qa/qa/support/formatters/quarantine_formatter.rb b/qa/qa/support/formatters/quarantine_formatter.rb index c5d16988dbd..4e4da99749c 100644 --- a/qa/qa/support/formatters/quarantine_formatter.rb +++ b/qa/qa/support/formatters/quarantine_formatter.rb @@ -18,7 +18,7 @@ module QA def example_group_started(example_group_notification) group = example_group_notification.group - skip_or_run_quarantined_tests_or_contexts(filters, group) + skip_or_run_quarantined_tests_or_contexts(group) end # Starts example @@ -28,13 +28,7 @@ module QA example = example_notification.example # if skip propagated from example_group, do not reset skip metadata - skip_or_run_quarantined_tests_or_contexts(filters, example) unless example.metadata[:skip] - end - - private - - def filters - @filters ||= ::RSpec.configuration.inclusion_filter.rules + skip_or_run_quarantined_tests_or_contexts(example) unless example.metadata[:skip] end end end diff --git a/qa/qa/support/formatters/test_stats_formatter.rb b/qa/qa/support/formatters/test_stats_formatter.rb index 6f6291b5856..7678cb8406c 100644 --- a/qa/qa/support/formatters/test_stats_formatter.rb +++ b/qa/qa/support/formatters/test_stats_formatter.rb @@ -84,7 +84,8 @@ module QA retry_attempts: example.metadata[:retry_attempts] || 0, job_url: QA::Runtime::Env.ci_job_url, pipeline_url: env('CI_PIPELINE_URL'), - pipeline_id: env('CI_PIPELINE_ID') + pipeline_id: env('CI_PIPELINE_ID'), + testcase: example.metadata[:testcase] } } rescue StandardError => e @@ -124,11 +125,11 @@ module QA @merge_request ||= (!!env('CI_MERGE_REQUEST_IID') || !!env('TOP_UPSTREAM_MERGE_REQUEST_IID')).to_s end - # Test run type from staging, canary, preprod or production env + # Test run type from staging (`gstg`, `gstg-cny`, `gstg-ref`), canary, preprod or production env # # @return [String, nil] def run_type - return unless %w[staging canary preprod production].include?(project_name) + return unless %w[staging staging-canary staging-ref canary preprod production].include?(project_name) @run_type ||= begin test_subset = if env('NO_ADMIN') == 'true' diff --git a/qa/qa/support/matchers/eventually_matcher.rb b/qa/qa/support/matchers/eventually_matcher.rb index dedef8e6b98..2fb5249d9af 100644 --- a/qa/qa/support/matchers/eventually_matcher.rb +++ b/qa/qa/support/matchers/eventually_matcher.rb @@ -28,27 +28,21 @@ module QA RSpec::Matchers.define(:"eventually_#{op}") do |*expected| chain(:within) do |kwargs = {}| @retry_args = kwargs - @retry_args[:sleep_interval] = 0.5 unless @retry_args[:sleep_interval] + @retry_args[:sleep_interval] = 0.5 unless kwargs[:sleep_interval] end - def supports_block_expectations? - true - end + description { "eventually #{operator_msg}: #{expected_formatted}" } match { |actual| wait_and_check(actual, :default_expectation) } match_when_negated { |actual| wait_and_check(actual, :when_negated_expectation) } - description do - "eventually #{operator_msg} #{expected.inspect}" - end + failure_message { fail_message } - failure_message do - "#{e}:\nexpected to #{description}, last attempt was #{@result.nil? ? 'nil' : @result}" - end + failure_message_when_negated { fail_message(negate: true) } - failure_message_when_negated do - "#{e}:\nexpected not to #{description}, last attempt was #{@result.nil? ? 'nil' : @result}" + def supports_block_expectations? + true end # Execute rspec expectation within retrier @@ -60,9 +54,9 @@ module QA attempt = 0 QA::Runtime::Logger.debug( - "Running eventually matcher with '#{operator_msg}' operator with: #{@retry_args}" + "Running eventually matcher with '#{operator_msg}' operator with: '#{retry_args}' arguments" ) - QA::Support::Retrier.retry_until(**@retry_args, log: false) do + QA::Support::Retrier.retry_until(**retry_args, log: false) do QA::Runtime::Logger.debug("evaluating expectation, attempt: #{attempt += 1}") public_send(expectation_name, actual) @@ -132,6 +126,44 @@ module QA [operator, expected] end end + + # Custom retry arguments + # + # @return [Hash] + def retry_args + @retry_args ||= { sleep_interval: 0.5 } + end + + # Custom failure message + # + # @param [Boolean] negate + # @return [String] + def fail_message(negate: false) + "#{e}:\n\nexpected #{negate ? 'not ' : ''}to #{description}\n\n"\ + "last attempt was: #{@result.nil? ? 'nil' : actual_formatted}\n\n"\ + "Diff:#{diff}" + end + + # Formatted expect + # + # @return [String] + def expected_formatted + RSpec::Support::ObjectFormatter.format(expected) + end + + # Formatted actual result + # + # @return [String] + def actual_formatted + RSpec::Support::ObjectFormatter.format(@result) + end + + # Object diff + # + # @return [String] + def diff + RSpec::Support::Differ.new(color: true).diff(@result, expected) + end end end end diff --git a/qa/qa/support/matchers/have_matcher.rb b/qa/qa/support/matchers/have_matcher.rb index 47d2d246460..a90d2df96ae 100644 --- a/qa/qa/support/matchers/have_matcher.rb +++ b/qa/qa/support/matchers/have_matcher.rb @@ -5,6 +5,7 @@ module QA module Matchers module HaveMatcher PREDICATE_TARGETS = %w[ + auto_devops_container element file_content assignee @@ -17,6 +18,8 @@ module QA package pipeline related_issue_item + sast_status + security_configuration_history_link snippet_description tag label diff --git a/qa/qa/support/page_error_checker.rb b/qa/qa/support/page_error_checker.rb new file mode 100644 index 00000000000..5d16245b4cd --- /dev/null +++ b/qa/qa/support/page_error_checker.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module QA + module Support + class PageErrorChecker + class << self + def report!(page, error_code) + report = if QA::Runtime::Env.browser == :chrome + return_chrome_errors(page, error_code) + else + status_code_report(error_code) + end + + raise "#{report}\n\n"\ + "Path: #{page.current_path}" + end + + def return_chrome_errors(page, error_code) + severe_errors = logs(page).select { |log| log.level == 'SEVERE' } + if severe_errors.none? + status_code_report(error_code) + else + "There #{severe_errors.count == 1 ? 'was' : 'were'} #{severe_errors.count} "\ + "SEVERE level error#{severe_errors.count == 1 ? '' : 's'}:\n\n#{error_report_for(severe_errors)}" + end + end + + def status_code_report(error_code) + "Status code #{error_code} found" + end + + def check_page_for_error_code(page) + error_code = 0 + # Test for 404 img alt + error_code = 404 if Nokogiri::HTML.parse(page.html).xpath("//img").map { |t| t[:alt] }.first.eql?('404') + + # 500 error page in header surrounded by newlines, try to match + five_hundred_test = Nokogiri::HTML.parse(page.html).xpath("//h1").map.first + unless five_hundred_test.nil? + error_code = 500 if five_hundred_test.text.include?('500') + end + # GDK shows backtrace rather than error page + error_code = 500 if Nokogiri::HTML.parse(page.html).xpath("//body//section").map { |t| t[:class] }.first.eql?('backtrace') + + unless error_code == 0 + report!(page, error_code) + end + end + + def error_report_for(logs) + logs + .map(&:message) + .map { |message| message.gsub('\\n', "\n") } + end + + def logs(page) + page.driver.browser.manage.logs.get(:browser) + end + end + end + end +end diff --git a/qa/qa/support/wait_for_requests.rb b/qa/qa/support/wait_for_requests.rb index 5109f51d4d7..16af4bae521 100644 --- a/qa/qa/support/wait_for_requests.rb +++ b/qa/qa/support/wait_for_requests.rb @@ -7,7 +7,12 @@ module QA DEFAULT_MAX_WAIT_TIME = 60 - def wait_for_requests(skip_finished_loading_check: false) + def wait_for_requests(skip_finished_loading_check: false, skip_resp_code_check: false) + # We have tests that use 404 pages, allow them to skip this check + unless skip_resp_code_check + QA::Support::PageErrorChecker.check_page_for_error_code(Capybara.page) + end + Waiter.wait_until(log: false) do finished_all_ajax_requests? && (!skip_finished_loading_check ? finished_loading?(wait: 1) : true) end diff --git a/qa/qa/tools/delete_projects.rb b/qa/qa/tools/delete_projects.rb index 240901eea6f..1f550f035d1 100644 --- a/qa/qa/tools/delete_projects.rb +++ b/qa/qa/tools/delete_projects.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_relative '../../qa' - # This script deletes all projects directly under a group specified by ENV['TOP_LEVEL_GROUP_NAME'] # Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS # Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group') diff --git a/qa/qa/tools/delete_subgroups.rb b/qa/qa/tools/delete_subgroups.rb index bc905fdeadd..11b45365d4c 100644 --- a/qa/qa/tools/delete_subgroups.rb +++ b/qa/qa/tools/delete_subgroups.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_relative '../../qa' - # This script deletes all subgroups of a group specified by ENV['TOP_LEVEL_GROUP_NAME'] # Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS # Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group') diff --git a/qa/qa/tools/delete_test_resources.rb b/qa/qa/tools/delete_test_resources.rb new file mode 100644 index 00000000000..917cb2fa992 --- /dev/null +++ b/qa/qa/tools/delete_test_resources.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +# This script reads from test_resources.txt file to collect data about resources to delete +# Deletes all deletable resources that E2E tests created +# Resource type: Sandbox, User, Fork and RSpec::Mocks::Double are not included +# +# Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS +# When in CI also requires: QA_TEST_RESOURCES_FILE_PATTERN +# Run `rake delete_test_resources[<file_pattern>]` + +module QA + module Tools + class DeleteTestResources + include Support::API + + def initialize(file_pattern = nil) + raise ArgumentError, "Please provide GITLAB_ADDRESS" unless ENV['GITLAB_ADDRESS'] + raise ArgumentError, "Please provide GITLAB_QA_ACCESS_TOKEN" unless ENV['GITLAB_QA_ACCESS_TOKEN'] + + @api_client = Runtime::API::Client.new(ENV['GITLAB_ADDRESS'], personal_access_token: ENV['GITLAB_QA_ACCESS_TOKEN']) + @file_pattern = file_pattern + end + + def run + puts 'Deleting test created resources...' + + if Runtime::Env.running_in_ci? + raise ArgumentError, 'Please provide QA_TEST_RESOURCES_FILE_PATTERN' unless ENV['QA_TEST_RESOURCES_FILE_PATTERN'] + + Dir.glob(@file_pattern).each do |file| + delete_resources(load_file(file)) + end + else + file = Runtime::Env.test_resources_created_filepath + raise ArgumentError, "'#{file}' either does not exist or empty." if !File.exist?(file) || File.zero?(file) + + delete_resources(load_file(file)) + end + + puts "\nDone" + end + + private + + def load_file(json) + JSON.parse(File.read(json)) + end + + def delete_resources(resources) + failures = [] + + resources.each_key do |type| + next if resources[type].empty? + + resources[type].each do |resource| + next if resource_not_found?(resource['api_path']) + + msg = resource['info'] ? "#{type} - #{resource['info']}" : "#{type} at #{resource['api_path']}" + + puts "\nDeleting #{msg}..." + delete_response = delete(Runtime::API::Request.new(@api_client, resource['api_path']).url) + + if delete_response.code == 202 + print "\e[32m.\e[0m" + else + print "\e[31mF\e[0m" + failures << msg + end + end + end + + unless failures.empty? + puts "\nFailed to delete #{failures.length} resources:\n" + puts failures + end + end + + def resource_not_found?(api_path) + get_response = get Runtime::API::Request.new(@api_client, api_path).url + + get_response.code.eql? 404 + end + end + end +end diff --git a/qa/qa/tools/delete_test_ssh_keys.rb b/qa/qa/tools/delete_test_ssh_keys.rb index 58ab4865336..9e5728a5509 100644 --- a/qa/qa/tools/delete_test_ssh_keys.rb +++ b/qa/qa/tools/delete_test_ssh_keys.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_relative '../../qa' - # This script deletes all selected test ssh keys for a specific user # Keys can be selected by a string matching part of the key's title and by created date # - Specify `title_portion` to delete only keys that include the string provided diff --git a/qa/qa/tools/generate_perf_testdata.rb b/qa/qa/tools/generate_perf_testdata.rb index 8e5da94e7e6..0f06fd2fbc4 100644 --- a/qa/qa/tools/generate_perf_testdata.rb +++ b/qa/qa/tools/generate_perf_testdata.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'yaml' -require_relative '../../qa' + # This script generates testdata for Performance Testing. # Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS # This job creates a urls.txt which contains a hash of all the URLs needed for Performance Testing diff --git a/qa/qa/tools/initialize_gitlab_auth.rb b/qa/qa/tools/initialize_gitlab_auth.rb index 3ead8fc9bd4..86791f1f624 100644 --- a/qa/qa/tools/initialize_gitlab_auth.rb +++ b/qa/qa/tools/initialize_gitlab_auth.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_relative '../../qa' - module QA module Tools # Task to set default password from Runtime::Env.default_password if not set already diff --git a/qa/qa/tools/knapsack_report.rb b/qa/qa/tools/knapsack_report.rb index fb405e82e83..e50c4fe63d2 100644 --- a/qa/qa/tools/knapsack_report.rb +++ b/qa/qa/tools/knapsack_report.rb @@ -5,50 +5,86 @@ require "fog/google" module QA module Tools class KnapsackReport + extend SingleForwardable + PROJECT = "gitlab-qa-resources" BUCKET = "knapsack-reports" + FALLBACK_REPORT = "knapsack/master_report.json" - class << self - def download - new.download_report - end + def_delegators :new, :configure!, :move_regenerated_report, :download_report, :upload_report - def upload(glob) - new.upload_report(glob) - end - end + # Configure knapsack report + # + # * Setup variables + # * Fetch latest report + # + # @return [void] + def configure! + ENV["KNAPSACK_TEST_FILE_PATTERN"] ||= "qa/specs/features/**/*_spec.rb" + ENV["KNAPSACK_REPORT_PATH"] = report_path - def initialize - ENV["KNAPSACK_REPORT_PATH"] || raise("KNAPSACK_REPORT_PATH env var is required!") - ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise("QA_KNAPSACK_REPORT_GCS_CREDENTIALS env var is required!") + Knapsack.logger = QA::Runtime::Logger.logger + + download_report end # Download knapsack report from gcs bucket # # @return [void] def download_report - logger.info("Downloading latest knapsack report '#{report_file}'") + logger.debug("Downloading latest knapsack report for '#{report_name}' to '#{report_path}'") file = client.get_object(BUCKET, report_file) - - logger.info("Saving latest knapsack report to '#{report_path}'") File.write(report_path, file[:body]) + rescue StandardError => e + ENV["KNAPSACK_REPORT_PATH"] = FALLBACK_REPORT + logger.warn("Failed to fetch latest knapsack report: #{e}") + logger.warn("Falling back to '#{FALLBACK_REPORT}'") + end + + # Rename and move new regenerated report to a separate folder used to indicate report name + # + # @return [void] + def move_regenerated_report + return unless ENV["KNAPSACK_GENERATE_REPORT"] == "true" + + tmp_path = "tmp/knapsack/#{report_name}" + FileUtils.mkdir_p(tmp_path) + + # Use path from knapsack config in case of fallback to master_report.json + knapsack_report_path = Knapsack.report.report_path + logger.debug("Moving regenerated #{knapsack_report_path} to save as artifact") + FileUtils.cp(knapsack_report_path, "#{tmp_path}/#{ENV['CI_NODE_INDEX']}.json") end # Merge and upload knapsack report to gcs bucket # + # Fetches all files defined in glob and uses parent folder as report name + # # @param [String] glob # @return [void] def upload_report(glob) - reports = Dir[glob] - return logger.error("Pattern '#{glob}' did not match any files!") if reports.empty? + reports = Pathname.glob(glob).each_with_object(Hash.new { |hsh, key| hsh[key] = [] }) do |report, hash| + next unless report.extname == ".json" + + hash[report.parent.basename.to_s].push(report) + end + return logger.error("Glob '#{glob}' did not contain any valid report files!") if reports.empty? - report = reports - .map { |path| JSON.parse(File.read(path)) } - .reduce({}, :merge) - return logger.error("Knapsack generated empty report, skipping upload!") if report.empty? + reports.each do |name, jsons| + file = "#{name}.json" - logger.info("Uploading latest knapsack report '#{report_file}'") - client.put_object(BUCKET, report_file, JSON.pretty_generate(report)) + report = jsons + .map { |json| JSON.parse(File.read(json)) } + .reduce({}, :merge) + .sort_by { |k, v| v } # sort report by execution time + .to_h + next logger.warn("Knapsack generated empty report for '#{name}', skipping upload!") if report.empty? + + logger.info("Uploading latest knapsack report '#{file}'") + client.put_object(BUCKET, file, JSON.pretty_generate(report)) + rescue StandardError => e + logger.error("Failed to upload knapsack report for '#{name}'. Error: #{e}") + end end private @@ -64,24 +100,50 @@ module QA # # @return [Fog::Storage::GoogleJSON] def client - @client ||= Fog::Storage::Google.new( - google_project: PROJECT, - google_json_key_location: ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] - ) + @client ||= Fog::Storage::Google.new(google_project: PROJECT, **gcs_credentials) + end + + # Base path of knapsack report + # + # @return [String] + def report_base_path + @report_base_path ||= "knapsack" end # Knapsack report path # # @return [String] def report_path - @report_path ||= ENV["KNAPSACK_REPORT_PATH"] + @report_path ||= "#{report_base_path}/#{report_file}" end # Knapsack report name # # @return [String] def report_file - @report_name ||= report_path.split("/").last + @report_file ||= "#{report_name}.json" + end + + # Report name + # + # Infer report name from ci job name + # Remove characters incompatible with gcs bucket naming from job names like ee:instance-parallel + # + # @return [String] + def report_name + @report_name ||= ENV["CI_JOB_NAME"].split(" ").first.tr(":", "-") + end + + # GCS credentials json + # + # @return [Hash] + def gcs_credentials + json_key = ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise( + "QA_KNAPSACK_REPORT_GCS_CREDENTIALS env variable is required!" + ) + return { google_json_key_location: json_key } if File.exist?(json_key) + + { google_json_key_string: json_key } end end end diff --git a/qa/qa/tools/long_running_spec_reporter.rb b/qa/qa/tools/long_running_spec_reporter.rb new file mode 100644 index 00000000000..ce035248baa --- /dev/null +++ b/qa/qa/tools/long_running_spec_reporter.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require "fog/google" +require "slack-notifier" + +module QA + module Tools + class LongRunningSpecReporter + extend SingleForwardable + + SLACK_CHANNEL = "#quality-reports" + PROJECT = "gitlab-qa-resources" + BUCKET = "knapsack-reports" + REPORT_NAME = "ee-instance-parallel.json" + RUNTIME_THRESHOLD = 300 + + def_delegator :new, :execute + + # Find and report specs exceeding runtime threshold + # + # @return [void] + def execute + return puts("No long running specs detected, all good!") if long_running_specs.empty? + + specs = long_running_specs.map { |k, v| "#{k}: #{(v / 60).round(2)} minutes" }.join("\n") + average = mean_runtime < 60 ? "#{mean_runtime.round(0)} seconds" : "#{(mean_runtime / 60).round(2)} minutes" + msg = <<~MSG + Following spec files are exceeding #{RUNTIME_THRESHOLD / 60} minute runtime threshold! + Current average spec runtime: #{average}. + MSG + + puts("#{msg}\n#{specs}") + notifier.post(icon_emoji: ":time-out:", text: "#{msg}\n```#{specs}```") + end + + private + + # Average runtime of spec files + # + # @return [Number] + def mean_runtime + @mean_runtime ||= latest_report.values + .select { |v| v < RUNTIME_THRESHOLD } + .yield_self { |runtimes| runtimes.sum(0.0) / runtimes.length } + end + + # Spec files exceeding runtime threshold + # + # @return [Hash] + def long_running_specs + @long_running_specs ||= latest_report.select { |k, v| v > RUNTIME_THRESHOLD } + end + + # Latest knapsack report + # + # @return [Hash] + def latest_report + @latest_report ||= JSON.parse(client.get_object(BUCKET, REPORT_NAME)[:body]) + end + + # Slack notifier + # + # @return [Slack::Notifier] + def notifier + @notifier ||= Slack::Notifier.new( + slack_webhook_url, + channel: SLACK_CHANNEL, + username: "Spec Runtime Report" + ) + end + + # GCS client + # + # @return [Fog::Storage::GoogleJSON] + def client + @client ||= Fog::Storage::Google.new( + google_project: PROJECT, + **(File.exist?(gcs_json) ? { google_json_key_location: gcs_json } : { google_json_key_string: gcs_json }) + ) + end + + # Slack webhook url + # + # @return [String] + def slack_webhook_url + @slack_webhook_url ||= ENV["SLACK_WEBHOOK"] || raise("Missing SLACK_WEBHOOK env variable") + end + + # GCS credentials json + # + # @return [Hash] + def gcs_json + ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise("Missing QA_KNAPSACK_REPORT_GCS_CREDENTIALS env variable!") + end + end + end +end diff --git a/qa/qa/tools/reliable_report.rb b/qa/qa/tools/reliable_report.rb index 40a452be36e..b99b97c1ea6 100644 --- a/qa/qa/tools/reliable_report.rb +++ b/qa/qa/tools/reliable_report.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_relative "../../qa" - require "influxdb-client" require "terminal-table" require "slack-notifier" @@ -16,7 +14,7 @@ module QA PROJECT_ID = 278964 def initialize(range) - @range = range + @range = range.to_i @influxdb_bucket = "e2e-test-stats" @slack_channel = "#quality-reports" @influxdb_url = ENV["QA_INFLUXDB_URL"] || raise("Missing QA_INFLUXDB_URL env variable") @@ -34,9 +32,8 @@ module QA reporter.print_report reporter.report_in_issue_and_slack if report_in_issue_and_slack == "true" rescue StandardError => e - puts "Report creation failed! Error: '#{e}'".colorize(:red) - reporter.notify_failure(e) - exit(1) + reporter&.notify_failure(e) + raise(e) end # Print top stable specs @@ -58,7 +55,11 @@ module QA puts "Creating report".colorize(:green) response = post( "#{gitlab_api_url}/projects/#{PROJECT_ID}/issues", - { title: "Reliable spec report", description: report_issue_body, labels: "Quality,test" }, + { + title: "Reliable e2e test report", + description: report_issue_body, + labels: "Quality,test,type::maintenance,reliable test report" + }, headers: { "PRIVATE-TOKEN" => gitlab_access_token } ) web_url = parse_body(response)[:web_url] @@ -96,68 +97,79 @@ module QA # # @return [String] def report_issue_body + execution_interval = "(#{Date.today - range} - #{Date.today})" + issue = [] issue << "[[_TOC_]]" - issue << "# Candidates for promotion to reliable\n\n```\n#{stable_summary_table}\n```" - issue << results_markdown(stable_results_tables) + issue << "# Candidates for promotion to reliable #{execution_interval}" + issue << "Total amount: **#{stable_test_runs.sum { |_k, v| v.count }}**" + issue << stable_summary_table(markdown: true).to_s + issue << results_markdown(:stable) return issue.join("\n\n") if unstable_reliable_test_runs.empty? - issue << "# Reliable specs with failures\n\n```\n#{unstable_summary_table}\n```" - issue << results_markdown(unstable_reliable_results_tables) + issue << "# Reliable specs with failures #{execution_interval}" + issue << "Total amount: **#{unstable_reliable_test_runs.sum { |_k, v| v.count }}**" + issue << unstable_summary_table(markdown: true).to_s + issue << results_markdown(:unstable) issue.join("\n\n") end # Stable spec summary table # + # @param [Boolean] markdown # @return [Terminal::Table] - def stable_summary_table - @stable_summary_table ||= terminal_table( + def stable_summary_table(markdown: false) + terminal_table( rows: stable_test_runs.map { |stage, specs| [stage, specs.length] }, title: "Stable spec summary for past #{range} days".ljust(50), - headings: %w[STAGE COUNT] + headings: %w[STAGE COUNT], + markdown: markdown ) end # Unstable reliable summary table # + # @param [Boolean] markdown # @return [Terminal::Table] - def unstable_summary_table - @unstable_summary_table ||= terminal_table( + def unstable_summary_table(markdown: false) + terminal_table( rows: unstable_reliable_test_runs.map { |stage, specs| [stage, specs.length] }, title: "Unstable spec summary for past #{range} days".ljust(50), - headings: %w[STAGE COUNT] + headings: %w[STAGE COUNT], + markdown: markdown ) end # Result tables for stable specs # + # @param [Boolean] markdown # @return [Hash] - def stable_results_tables - @stable_results ||= results_tables(:stable) + def stable_results_tables(markdown: false) + results_tables(:stable, markdown: markdown) end # Result table for unstable specs # + # @param [Boolean] markdown # @return [Hash] - def unstable_reliable_results_tables - @unstable_results ||= results_tables(:unstable) + def unstable_reliable_results_tables(markdown: false) + results_tables(:unstable, markdown: markdown) end # Markdown formatted tables # - # @param [Hash] results + # @param [Symbol] type result type - :stable, :unstable # @return [String] - def results_markdown(results) - results.map do |stage, table| + def results_markdown(type) + runs = type == :stable ? stable_test_runs : unstable_reliable_test_runs + results_tables(type, markdown: true).map do |stage, table| <<~STAGE.strip - ## #{stage} + ## #{stage} (#{runs[stage].count}) <details> <summary>Executions table</summary> - ``` #{table} - ``` </details> STAGE @@ -167,15 +179,19 @@ module QA # Results table # # @param [Symbol] type result type - :stable, :unstable + # @param [Boolean] markdown # @return [Hash<Symbol, Terminal::Table>] - def results_tables(type) + def results_tables(type, markdown: false) (type == :stable ? stable_test_runs : unstable_reliable_test_runs).to_h do |stage, specs| headings = ["name", "runs", "failures", "failure rate"] [stage, terminal_table( - rows: specs.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] }, title: "Top #{type} specs in '#{stage}' stage for past #{range} days", - headings: headings.map(&:upcase) + headings: headings.map(&:upcase), + markdown: markdown, + rows: specs.map do |k, v| + [name_column(name: k, file: v[:file], markdown: markdown), *table_params(v.values)] + end )] end end @@ -214,13 +230,17 @@ module QA # Terminal table for result formatting # + # @param [Array] rows + # @param [Array] headings + # @param [String] title + # @param [Boolean] markdown # @return [Terminal::Table] - def terminal_table(rows:, headings:, title: nil) + def terminal_table(rows:, headings:, title:, markdown:) Terminal::Table.new( headings: headings, - style: { all_separators: true }, - title: title, - rows: rows + title: markdown ? nil : title, + rows: rows, + style: markdown ? { border: :markdown } : { all_separators: true } ) end @@ -232,17 +252,17 @@ module QA [*parameters[1..2], "#{parameters.last}%"] end - # Name column value + # Name column content # # @param [String] name # @param [String] file + # @param [Boolean] markdown # @return [String] - def name_column(name, file) - spec_name = name.length > 150 ? "#{name} ".scan(/.{1,150} /).map(&:strip).join("\n") : name - name_line = "name: '#{spec_name}'" - file_line = "file: '#{file}'" + def name_column(name:, file:, markdown: false) + return "**name**: #{name}<br>**file**: #{file}" if markdown - "#{name_line}\n#{file_line.ljust(160)}" + wrapped_name = name.length > 150 ? "#{name} ".scan(/.{1,150} /).map(&:strip).join("\n") : name + "name: '#{wrapped_name}'\nfile: #{file.ljust(160)}" end # Test executions grouped by name @@ -254,10 +274,16 @@ module QA all_runs = query_api.query(query: query(reliable)).values all_runs.each_with_object(Hash.new { |hsh, key| hsh[key] = {} }) do |table, result| - records = table.records - name = records.last.values["name"] - file = records.last.values["file_path"].split("/").last - stage = records.last.values["stage"] || "unknown" + records = table.records.sort_by { |record| record.values["_time"] } + # skip specs that executed less time than defined by range or stopped executing before report date + # offset 1 day due to how schedulers are configured and first run can be 1 day later + next if (Date.today - Date.parse(records.first.values["_time"])).to_i < (range - 1) + next if (Date.today - Date.parse(records.last.values["_time"])).to_i > 1 + + last_record = records.last.values + name = last_record["name"] + file = last_record["file_path"].split("/").last + stage = last_record["stage"] || "unknown" runs = records.count failed = records.count { |r| r.values["status"] == "failed" } diff --git a/qa/qa/tools/revoke_all_personal_access_tokens.rb b/qa/qa/tools/revoke_all_personal_access_tokens.rb index c0a1697fa16..b4fa02a36d4 100644 --- a/qa/qa/tools/revoke_all_personal_access_tokens.rb +++ b/qa/qa/tools/revoke_all_personal_access_tokens.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -require_relative '../../qa' require 'net/protocol' + # This script revokes all personal access tokens with the name of 'api-test-token' on the host specified by GITLAB_ADDRESS # Required environment variables: GITLAB_USERNAME, GITLAB_PASSWORD and GITLAB_ADDRESS # Run `rake revoke_personal_access_tokens` diff --git a/qa/qa/tools/test_resource_data_processor.rb b/qa/qa/tools/test_resource_data_processor.rb new file mode 100644 index 00000000000..78fb6ef6cd0 --- /dev/null +++ b/qa/qa/tools/test_resource_data_processor.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +# This script collects all resources created during each test execution +# Save the data and write it to a JSON file at the end of suite + +module QA + module Tools + class TestResourceDataProcessor + @resources ||= Hash.new { |hsh, key| hsh[key] = [] } + + class << self + # Ignoring rspec-mocks, sandbox, user and fork resources + # TODO: Will need to figure out which user resources can be collected, ignore for now + # + # Collecting resources created in E2E tests + # Data is a Hash of resources with keys as resource type (group, project, issue, etc.) + # Each type contains an array of resource object (hash) of the same type + # E.g: { "QA::Resource::Project": [ { info: 'foo', api_path: '/foo'}, {...} ] } + def collect(resource, info) + return if resource.api_response.nil? || + resource.is_a?(RSpec::Mocks::Double) || + resource.is_a?(Resource::Sandbox) || + resource.is_a?(Resource::User) || + resource.is_a?(Resource::Fork) + + api_path = if resource.respond_to?(:api_delete_path) + resource.api_delete_path.gsub('%2F', '/') + elsif resource.respond_to?(:api_get_path) + resource.api_get_path.gsub('%2F', '/') + else + 'Cannot find resource API path' + end + + type = resource.class.name + + @resources[type] << { info: info, api_path: api_path } + end + + # If JSON file exists and not empty, read and load file content + # Merge what is saved in @resources into the content from file + # Overwrite file content with the new data hash + # Otherwise create file and write data hash to file for the first time + def write_to_file + return if @resources.empty? + + file = Runtime::Env.test_resources_created_filepath + FileUtils.mkdir_p('tmp/') + FileUtils.touch(file) + data = nil + + if File.zero?(file) + data = @resources + else + data = JSON.parse(File.read(file)) + + @resources.each_pair do |key, val| + data[key].nil? ? data[key] = val : val.each { |item| data[key] << item } + end + end + + File.open(file, 'w') { |f| f.write(JSON.pretty_generate(data.each_value(&:uniq!))) } + end + end + end + end +end |