diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-11-18 13:16:36 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-11-18 13:16:36 +0000 |
| commit | 311b0269b4eb9839fa63f80c8d7a58f32b8138a0 (patch) | |
| tree | 07e7870bca8aed6d61fdcc810731c50d2c40af47 /qa | |
| parent | 27909cef6c4170ed9205afa7426b8d3de47cbb0c (diff) | |
| download | gitlab-ce-311b0269b4eb9839fa63f80c8d7a58f32b8138a0.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-5-stable-eev14.5.0-rc42
Diffstat (limited to 'qa')
125 files changed, 2864 insertions, 1097 deletions
diff --git a/qa/Gemfile b/qa/Gemfile index ee90d049d7b..498d05b2254 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -7,9 +7,9 @@ gem 'activesupport', '~> 6.1.4.1' # This should stay in sync with the root's Gem gem 'allure-rspec', '~> 2.15.0' gem 'capybara', '~> 3.35.0' gem 'capybara-screenshot', '~> 1.0.23' -gem 'rake', '~> 12.3.3' +gem 'rake', '~> 13' gem 'rspec', '~> 3.10' -gem 'selenium-webdriver', '~> 4.0.0.rc1' +gem 'selenium-webdriver', '~> 4.0' gem 'airborne', '~> 0.3.4', require: false # airborne is messing with rspec sandboxed mode so not requiring by default gem 'rest-client', '~> 2.1.0' gem 'rspec-retry', '~> 0.6.1', require: 'rspec/retry' @@ -22,9 +22,11 @@ gem 'timecop', '~> 0.9.1' gem 'parallel', '~> 1.19' gem 'rspec-parameterized', '~> 0.4.2' gem 'octokit', '~> 4.21' -gem 'webdrivers', '~> 4.6' +gem 'webdrivers', '~> 5.0' gem 'zeitwerk', '~> 2.4' gem 'influxdb-client', '~> 1.17' +gem 'terminal-table', '~> 1.8', require: false +gem 'slack-notifier', '~> 2.4', require: false gem 'chemlab', '~> 0.9' gem 'chemlab-library-www-gitlab-com', '~> 0.1' diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 153a141d3fd..2b5b5e368cf 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -11,7 +11,7 @@ GEM adamantium (0.2.0) ice_nine (~> 0.11.0) memoizable (~> 0.4.0) - addressable (2.7.0) + addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) airborne (0.3.4) activesupport @@ -27,7 +27,7 @@ GEM oj (>= 3.10, < 4) require_all (>= 2, < 4) uuid (>= 2.3, < 3) - ast (2.4.1) + ast (2.4.2) binding_ninja (0.2.3) byebug (9.1.0) capybara (3.35.3) @@ -41,7 +41,7 @@ GEM capybara-screenshot (1.0.23) capybara (>= 1.0, < 4) launchy - chemlab (0.9.1) + chemlab (0.9.2) colorize (~> 0.8) i18n (~> 1.8) rake (>= 12, < 14) @@ -88,27 +88,25 @@ GEM gitlab (4.16.1) httparty (~> 0.14, >= 0.14.0) terminal-table (~> 1.5, >= 1.5.1) - gitlab-qa (7.9.1) + gitlab-qa (7.14.0) activesupport (~> 6.1) gitlab (~> 4.16.1) - http (= 4.3.0) + http (~> 5.0) nokogiri (~> 1.10) table_print (= 1.5.7) - http (4.3.0) - addressable (~> 2.3) + http (5.0.4) + addressable (~> 2.8) http-cookie (~> 1.0) http-form_data (~> 2.2) - http-parser (~> 1.2.0) + llhttp-ffi (~> 0.4.0) http-accept (1.7.0) - http-cookie (1.0.3) + http-cookie (1.0.4) domain_name (~> 0.5) http-form_data (2.3.0) - http-parser (1.2.3) - ffi-compiler (>= 1.0, < 2.0) - httparty (0.19.0) + httparty (0.20.0) mime-types (~> 3.0) multi_xml (>= 0.5.2) - i18n (1.8.10) + i18n (1.8.11) concurrent-ruby (~> 1.0) ice_nine (0.11.2) influxdb-client (1.17.0) @@ -116,22 +114,25 @@ GEM rake launchy (2.4.3) addressable (~> 2.3) + llhttp-ffi (0.4.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) macaddr (1.7.2) systemu (~> 2.6.5) memoizable (0.4.2) thread_safe (~> 0.3, >= 0.3.1) method_source (0.9.0) - mime-types (3.3.1) + mime-types (3.4.0) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0704) + mime-types-data (3.2021.1115) mini_mime (1.1.0) - mini_portile2 (2.5.3) + mini_portile2 (2.6.1) minitest (5.14.4) multi_xml (0.6.0) multipart-post (2.1.1) netrc (0.11.0) - nokogiri (1.11.7) - mini_portile2 (~> 2.5.0) + nokogiri (1.12.5) + mini_portile2 (~> 2.6.1) racc (~> 1.4) octokit (4.21.0) faraday (>= 0.9) @@ -140,7 +141,7 @@ GEM parallel (1.19.2) parallel_tests (2.29.0) parallel - parser (2.7.1.4) + parser (3.0.2.0) ast (~> 2.4.1) proc_to_ast (0.1.0) coderay @@ -153,12 +154,12 @@ GEM pry-byebug (3.5.1) byebug (~> 9.1) pry (~> 0.10) - public_suffix (4.0.1) - racc (1.5.2) + public_suffix (4.0.6) + racc (1.6.0) rack (2.2.3) rack-test (1.1.0) rack (>= 1.0, < 3) - rake (12.3.3) + rake (13.0.6) regexp_parser (2.1.1) require_all (3.0.0) rest-client (2.1.0) @@ -198,10 +199,11 @@ GEM sawyer (0.8.2) addressable (>= 2.3.5) faraday (> 0.8, < 2.0) - selenium-webdriver (4.0.0.rc1) + selenium-webdriver (4.0.3) childprocess (>= 0.5, < 5.0) - rexml (~> 3.2) + rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2) + slack-notifier (2.4.0) systemu (2.6.5) table_print (1.5.7) terminal-table (1.8.0) @@ -212,7 +214,7 @@ GEM concurrent-ruby (~> 1.0) unf (0.1.4) unf_ext - unf_ext (0.0.7.7) + unf_ext (0.0.8) unicode-display_width (1.8.0) unparser (0.4.7) abstract_type (~> 0.0.7) @@ -227,13 +229,13 @@ GEM watir (6.19.1) regexp_parser (>= 1.2, < 3) selenium-webdriver (>= 3.142.7) - webdrivers (4.6.0) + webdrivers (5.0.0) nokogiri (~> 1.6) rubyzip (>= 1.3.0) - selenium-webdriver (>= 3.0, < 4.0) + selenium-webdriver (~> 4.0) xpath (3.2.0) nokogiri (~> 1.8) - zeitwerk (2.4.2) + zeitwerk (2.5.1) PLATFORMS ruby @@ -255,7 +257,7 @@ DEPENDENCIES parallel (~> 1.19) parallel_tests (~> 2.29) pry-byebug (~> 3.5.1) - rake (~> 12.3.3) + rake (~> 13) rest-client (~> 2.1.0) rotp (~> 3.1.0) rspec (~> 3.10) @@ -263,10 +265,12 @@ DEPENDENCIES rspec-retry (~> 0.6.1) rspec_junit_formatter (~> 0.4.1) ruby-debug-ide (~> 0.7.0) - selenium-webdriver (~> 4.0.0.rc1) + selenium-webdriver (~> 4.0) + slack-notifier (~> 2.4) + terminal-table (~> 1.8) timecop (~> 0.9.1) - webdrivers (~> 4.6) + webdrivers (~> 5.0) zeitwerk (~> 2.4) BUNDLED WITH - 2.2.22 + 2.2.30 diff --git a/qa/Rakefile b/qa/Rakefile index f24c81a9ec2..57360e98ca2 100644 --- a/qa/Rakefile +++ b/qa/Rakefile @@ -2,6 +2,7 @@ # rubocop:disable Rails/RakeEnvironment load 'tasks/webdrivers.rake' +load 'tasks/reliable_report.rake' require_relative 'qa/tools/revoke_all_personal_access_tokens' require_relative 'qa/tools/delete_subgroups' diff --git a/qa/chemlab-library-gitlab.gemspec b/qa/chemlab-library-gitlab.gemspec index 34a55ba8927..9af4a650d98 100644 --- a/qa/chemlab-library-gitlab.gemspec +++ b/qa/chemlab-library-gitlab.gemspec @@ -19,4 +19,5 @@ Gem::Specification.new do |spec| spec.require_paths = ['lib'] spec.add_runtime_dependency 'chemlab', '~> 0.9' + spec.add_runtime_dependency 'zeitwerk', '~> 2.4' end diff --git a/qa/lib/gitlab.rb b/qa/lib/gitlab.rb index 4418e51facb..8c33071633d 100644 --- a/qa/lib/gitlab.rb +++ b/qa/lib/gitlab.rb @@ -1,31 +1,14 @@ # frozen_string_literal: true require 'chemlab/library' +require 'zeitwerk' + +loader = Zeitwerk::Loader.new +loader.push_dir(__dir__) +loader.ignore("#{__dir__}/gitlab/**/*.stub.rb") # ignore page stubs +loader.setup # Chemlab Page Libraries for GitLab module Gitlab include Chemlab::Library - - module Page - module Main - autoload :Login, 'gitlab/page/main/login' - autoload :SignUp, 'gitlab/page/main/sign_up' - end - - module Subscriptions - autoload :New, 'gitlab/page/subscriptions/new' - end - - module Admin - autoload :Dashboard, 'gitlab/page/admin/dashboard' - autoload :Subscription, 'gitlab/page/admin/subscription' - end - - module Group - module Settings - autoload :Billing, 'gitlab/page/group/settings/billing' - autoload :UsageQuotas, 'gitlab/page/group/settings/usage_quotas' - end - end - end end diff --git a/qa/lib/gitlab/page/admin/subscription.rb b/qa/lib/gitlab/page/admin/subscription.rb index 0f7c6b4c211..cdd9bb20b42 100644 --- a/qa/lib/gitlab/page/admin/subscription.rb +++ b/qa/lib/gitlab/page/admin/subscription.rb @@ -6,7 +6,16 @@ module Gitlab class Subscription < Chemlab::Page path '/admin/subscription' + p :plan + p :started + p :name + p :company + p :email + h2 :billable_users + h2 :maximum_users h2 :users_in_subscription + h2 :users_over_subscription + table :subscription_history end end end diff --git a/qa/lib/gitlab/page/admin/subscription.stub.rb b/qa/lib/gitlab/page/admin/subscription.stub.rb index 51f23e7f0d0..89d7bfb95d9 100644 --- a/qa/lib/gitlab/page/admin/subscription.stub.rb +++ b/qa/lib/gitlab/page/admin/subscription.stub.rb @@ -4,6 +4,174 @@ module Gitlab module Page module Admin module Subscription + # @note Defined as +p :plan+ + # @return [String] The text content or value of +plan+ + def plan + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription.plan_element).to exist + # end + # @return [Watir::P] The raw +P+ element + def plan_element + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription).to be_plan + # end + # @return [Boolean] true if the +plan+ element is present on the page + def plan? + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @note Defined as +p :started+ + # @return [String] The text content or value of +started+ + def started + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription.started_element).to exist + # end + # @return [Watir::P] The raw +P+ element + def started_element + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription).to be_started + # end + # @return [Boolean] true if the +started+ element is present on the page + def started? + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @note Defined as +p :name+ + # @return [String] The text content or value of +name+ + def name + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription.name_element).to exist + # end + # @return [Watir::P] The raw +P+ element + def name_element + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription).to be_name + # end + # @return [Boolean] true if the +name+ element is present on the page + def name? + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @note Defined as +p :company+ + # @return [String] The text content or value of +company+ + def company + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription.company_element).to exist + # end + # @return [Watir::P] The raw +P+ element + def company_element + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription).to be_company + # end + # @return [Boolean] true if the +company+ element is present on the page + def company? + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @note Defined as +p :email+ + # @return [String] The text content or value of +email+ + def email + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription.email_element).to exist + # end + # @return [Watir::P] The raw +P+ element + def email_element + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription).to be_email + # end + # @return [Boolean] true if the +email+ element is present on the page + def email? + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @note Defined as +h2 :billable_users+ + # @return [String] The text content or value of +billable_users+ + def billable_users + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription.billable_users_element).to exist + # end + # @return [Watir::H2] The raw +H2+ element + def billable_users_element + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription).to be_billable_users + # end + # @return [Boolean] true if the +billable_users+ element is present on the page + def billable_users? + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @note Defined as +h2 :maximum_users+ + # @return [String] The text content or value of +maximum_users+ + def maximum_users + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription.maximum_users_element).to exist + # end + # @return [Watir::H2] The raw +H2+ element + def maximum_users_element + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription).to be_maximum_users + # end + # @return [Boolean] true if the +maximum_users+ element is present on the page + def maximum_users? + # This is a stub, used for indexing. The method is dynamically generated. + end + # @note Defined as +h2 :users_in_subscription+ # @return [String] The text content or value of +users_in_subscription+ def users_in_subscription @@ -27,6 +195,54 @@ module Gitlab def users_in_subscription? # This is a stub, used for indexing. The method is dynamically generated. end + + # @note Defined as +h2 :users_over_subscription+ + # @return [String] The text content or value of +users_over_subscription+ + def users_over_subscription + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription.users_over_subscription_element).to exist + # end + # @return [Watir::H2] The raw +H2+ element + def users_over_subscription_element + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription).to be_users_over_subscription + # end + # @return [Boolean] true if the +users_over_subscription+ element is present on the page + def users_over_subscription? + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @note Defined as +table :subscription_history+ + # @return [String] The text content or value of +subscription_history+ + def subscription_history + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription.subscription_history_element).to exist + # end + # @return [Watir::Table] The raw +Table+ element + def subscription_history_element + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Admin::Subscription.perform do |subscription| + # expect(subscription).to be_subscription_history + # end + # @return [Boolean] true if the +subscription_history+ element is present on the page + def subscription_history? + # This is a stub, used for indexing. The method is dynamically generated. + end end end end diff --git a/qa/lib/gitlab/page/main/welcome.rb b/qa/lib/gitlab/page/main/welcome.rb new file mode 100644 index 00000000000..a2df1da61c9 --- /dev/null +++ b/qa/lib/gitlab/page/main/welcome.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Page + module Main + class Welcome < Chemlab::Page + path '/users/sign_up/welcome' + + button :get_started_button + end + end + end +end diff --git a/qa/lib/gitlab/page/main/welcome.stub.rb b/qa/lib/gitlab/page/main/welcome.stub.rb new file mode 100644 index 00000000000..a10e697bcbf --- /dev/null +++ b/qa/lib/gitlab/page/main/welcome.stub.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Page + module Main + module Welcome + # @note Defined as +button :get_started_button+ + # Clicks +get_started_button+ + def get_started_button + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Main::Welcome.perform do |welcome| + # expect(welcome.get_started_button_element).to exist + # end + # @return [Watir::Button] The raw +Button+ element + def get_started_button_element + # This is a stub, used for indexing. The method is dynamically generated. + end + + # @example + # Gitlab::Page::Main::Welcome.perform do |welcome| + # expect(welcome).to be_get_started_button + # end + # @return [Boolean] true if the +get_started_button+ element is present on the page + def get_started_button? + # This is a stub, used for indexing. The method is dynamically generated. + end + end + end + end +end @@ -50,7 +50,8 @@ module QA "user_gpg" => "UserGPG", "smtp" => "SMTP", "otp" => "OTP", - "jira_api" => "JiraAPI" + "jira_api" => "JiraAPI", + "registry_tls" => "RegistryTLS" ) loader.setup diff --git a/qa/qa/fixtures/kubernetes_agent/agentk-manifest.yaml.erb b/qa/qa/fixtures/kubernetes_agent/agentk-manifest.yaml.erb index a13c92d5c6d..8eac8419022 100644 --- a/qa/qa/fixtures/kubernetes_agent/agentk-manifest.yaml.erb +++ b/qa/qa/fixtures/kubernetes_agent/agentk-manifest.yaml.erb @@ -24,8 +24,7 @@ spec: args: - --token-file=/config/token - --kas-address - - "<%= kas_wss_address %>" # Use this for GitLab chart deployments - # - "<%= kas_grpc_address %>" # Use this for GDK + - "<%= kas_wss_address %>" volumeMounts: - name: token-volume mountPath: /config diff --git a/qa/qa/fixtures/rubygems_package/mygem.gemspec b/qa/qa/fixtures/rubygems_package/mygem.gemspec deleted file mode 100644 index 33d8c88e5ac..00000000000 --- a/qa/qa/fixtures/rubygems_package/mygem.gemspec +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -Gem::Specification.new do |s| - s.name = 'mygem' - s.authors = ['Tanuki Steve', 'Hal 9000'] - s.author = 'Tanuki Steve' - s.version = '0.0.1' - s.date = '2011-09-29' - s.summary = 'package is the best' - s.files = ['lib/hello_gem.rb'] - s.require_paths = ['lib'] - - s.description = 'A test package for GitLab.' - s.email = 'tanuki@not_real.com' - s.homepage = 'https://gitlab.com/ruby-co/my-package' - s.license = 'MIT' - - s.metadata = { - 'bug_tracker_uri' => 'https://gitlab.com/ruby-co/my-package/issues', - 'changelog_uri' => 'https://gitlab.com/ruby-co/my-package/CHANGELOG.md', - 'documentation_uri' => 'https://gitlab.com/ruby-co/my-package/docs', - 'mailing_list_uri' => 'https://gitlab.com/ruby-co/my-package/mailme', - 'source_code_uri' => 'https://gitlab.com/ruby-co/my-package' - } - - s.bindir = 'bin' - s.platform = Gem::Platform::RUBY - s.post_install_message = 'Installed, thank you!' - s.rdoc_options = ['--main'] - s.required_ruby_version = '>= 2.7.0' - s.required_rubygems_version = '>= 1.8.11' - s.requirements = 'A high powered server or calculator' - s.rubygems_version = '1.8.09' - - s.add_dependency 'dependency_1', '~> 1.2.3' - s.add_dependency 'dependency_2', '3.0.0' - s.add_dependency 'dependency_3', '>= 1.0.0' - s.add_dependency 'dependency_4' -end diff --git a/qa/qa/flow/login.rb b/qa/qa/flow/login.rb index 05a509588f1..5f7e0227ac5 100644 --- a/qa/qa/flow/login.rb +++ b/qa/qa/flow/login.rb @@ -23,8 +23,11 @@ module QA end def sign_in(as: nil, address: :gitlab, skip_page_validation: false, admin: false) - Page::Main::Menu.perform(&:sign_out) if Page::Main::Menu.perform(&:signed_in?) - Runtime::Browser.visit(address, Page::Main::Login) + unless Page::Main::Login.perform(&:on_login_page?) + Page::Main::Menu.perform(&:sign_out) if Page::Main::Menu.perform(&:signed_in?) + Runtime::Browser.visit(address, Page::Main::Login) + end + Page::Main::Login.perform do |login| if admin login.sign_in_using_admin_credentials diff --git a/qa/qa/mobile/page/main/menu.rb b/qa/qa/mobile/page/main/menu.rb new file mode 100644 index 00000000000..40bb421b383 --- /dev/null +++ b/qa/qa/mobile/page/main/menu.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module QA + module Mobile + module Page + module Main + module Menu + extend QA::Page::PageConcern + + def self.prepended(base) + super + + base.class_eval do + view 'app/views/layouts/header/_default.html.haml' do + element :mobile_navbar_button, required: true + end + + view 'app/assets/javascripts/nav/components/responsive_home.vue' do + element :mobile_new_dropdown + end + end + end + + def open_mobile_menu + if has_no_element?(:user_avatar) + Support::Retrier.retry_until do + click_element(:mobile_navbar_button) + has_element?(:user_avatar) + end + end + end + + def open_mobile_new_dropdown + open_mobile_menu + + Support::Retrier.retry_until do + find('[data-qa-selector="mobile_new_dropdown"] > button').click + has_css?('.dropdown-menu-right.show') + end + end + + def has_personal_area?(wait: Capybara.default_max_wait_time) + open_mobile_menu + super + end + + def has_no_personal_area?(wait: Capybara.default_max_wait_time) + open_mobile_menu + super + end + + def within_user_menu + open_mobile_menu + super + end + end + end + end + end +end diff --git a/qa/qa/mobile/page/profile/menu.rb b/qa/qa/mobile/page/profile/menu.rb new file mode 100644 index 00000000000..34c53a95e03 --- /dev/null +++ b/qa/qa/mobile/page/profile/menu.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module QA + module Mobile + module Page + module Profile + module Menu + extend QA::Page::PageConcern + + def self.prepended(base) + super + + base.class_eval do + prepend QA::Mobile::Page::Main::Menu + end + end + + def within_sidebar + open_mobile_nav_sidebar + super + end + end + end + end + end +end diff --git a/qa/qa/mobile/page/project/issue/show.rb b/qa/qa/mobile/page/project/issue/show.rb new file mode 100644 index 00000000000..017ecebcb69 --- /dev/null +++ b/qa/qa/mobile/page/project/issue/show.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module QA + module Mobile + module Page + module Project + module Issue + module Show + extend QA::Page::PageConcern + + def self.prepended(base) + super + + base.class_eval do + view 'app/assets/javascripts/issue_show/components/header_actions.vue' do + element :issue_actions_dropdown + element :mobile_close_issue_button + element :mobile_reopen_issue_button + end + end + end + + def click_close_issue_button + find('[data-qa-selector="issue_actions_dropdown"] > button').click + find_element(:mobile_close_issue_button, visible: false).click + end + + def has_reopen_issue_button? + find('[data-qa-selector="issue_actions_dropdown"] > button').click + has_element?(:mobile_reopen_issue_button) + end + end + end + end + end + end +end diff --git a/qa/qa/mobile/page/project/show.rb b/qa/qa/mobile/page/project/show.rb new file mode 100644 index 00000000000..8a0a222c852 --- /dev/null +++ b/qa/qa/mobile/page/project/show.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module QA + module Mobile + module Page + module Project + module Show + extend QA::Page::PageConcern + + def self.prepended(base) + super + + base.class_eval do + prepend QA::Mobile::Page::Main::Menu + + view 'app/assets/javascripts/nav/components/top_nav_new_dropdown.vue' do + element :new_issue_mobile_button + end + end + end + + def go_to_new_issue + open_mobile_new_dropdown + + click_element(:new_issue_mobile_button) + end + end + end + end + end +end diff --git a/qa/qa/mobile/page/sub_menus/common.rb b/qa/qa/mobile/page/sub_menus/common.rb new file mode 100644 index 00000000000..6a0477a3f31 --- /dev/null +++ b/qa/qa/mobile/page/sub_menus/common.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module QA + module Mobile + module Page + module SubMenus + module Common + def open_mobile_nav_sidebar + if has_element?(:project_sidebar, visible: false) + Support::Retrier.retry_until do + click_element(:toggle_mobile_nav_button) + has_element?(:project_sidebar, visible: true) + end + end + end + + def within_sidebar + wait_for_requests + + open_mobile_nav_sidebar + + super + end + end + end + end + end +end diff --git a/qa/qa/page/base.rb b/qa/qa/page/base.rb index 9debdc1d4dd..4708063b2eb 100644 --- a/qa/qa/page/base.rb +++ b/qa/qa/page/base.rb @@ -57,16 +57,25 @@ module QA end end - def retry_until(max_attempts: 3, reload: false, sleep_interval: 0, raise_on_failure: true) - Support::Retrier.retry_until(max_attempts: max_attempts, reload_page: (reload && self), sleep_interval: sleep_interval, raise_on_failure: raise_on_failure) do - yield - end - end - - def retry_on_exception(max_attempts: 3, reload: false, sleep_interval: 0.5) - Support::Retrier.retry_on_exception(max_attempts: max_attempts, reload_page: (reload && self), sleep_interval: sleep_interval) do - yield - end + def retry_until(max_attempts: 3, reload: false, sleep_interval: 0, raise_on_failure: true, message: nil, &block) + Support::Retrier.retry_until( + max_attempts: max_attempts, + reload_page: (reload && self), + sleep_interval: sleep_interval, + raise_on_failure: raise_on_failure, + message: message, + &block + ) + end + + def retry_on_exception(max_attempts: 3, reload: false, sleep_interval: 0.5, message: nil, &block) + Support::Retrier.retry_on_exception( + max_attempts: max_attempts, + reload_page: (reload && self), + sleep_interval: sleep_interval, + message: message, + &block + ) end def scroll_to(selector, text: nil) diff --git a/qa/qa/page/component/issuable/sidebar.rb b/qa/qa/page/component/issuable/sidebar.rb index 971e7634f6d..77962570aed 100644 --- a/qa/qa/page/component/issuable/sidebar.rb +++ b/qa/qa/page/component/issuable/sidebar.rb @@ -22,20 +22,16 @@ module QA element :labels_block end - base.view 'app/assets/javascripts/vue_shared/components/sidebar/labels_select_vue/dropdown_value.vue' do - element :selected_label_content + base.view 'app/assets/javascripts/vue_shared/components/sidebar/labels_select_vue/dropdown_contents_labels_view.vue' do + element :dropdown_input_field end - base.view 'app/assets/javascripts/vue_shared/components/sidebar/labels_select_vue/dropdown_contents.vue' do + base.view 'app/assets/javascripts/vue_shared/components/sidebar/labels_select_widget/dropdown_contents.vue' do element :labels_dropdown_content end - base.view 'app/assets/javascripts/vue_shared/components/sidebar/labels_select_vue/dropdown_title.vue' do - element :labels_edit_button - end - - base.view 'app/assets/javascripts/vue_shared/components/sidebar/labels_select_vue/dropdown_contents_labels_view.vue' do - element :dropdown_input_field + base.view 'app/assets/javascripts/vue_shared/components/sidebar/labels_select_widget/dropdown_value.vue' do + element :selected_label_content end base.view 'app/views/shared/issuable/_sidebar.html.haml' do @@ -53,7 +49,7 @@ module QA end def assign_milestone(milestone) - within_element(:milestone_block) do + wait_milestone_block_finish_loading do click_element(:edit_link) click_on(milestone.title) end @@ -70,14 +66,14 @@ module QA end def has_assignee?(username) - within_element(:assignee_block) do - has_text?(username, wait: 1) + wait_assignees_block_finish_loading do + has_text?(username) end end def has_no_assignee?(username) - within_element(:assignee_block) do - has_no_text?(username, wait: 1) + wait_assignees_block_finish_loading do + has_no_text?(username) end end @@ -88,8 +84,14 @@ module QA end def has_label?(label) - within_element(:labels_block) do - !!has_element?(:selected_label_content, label_name: label) + wait_labels_block_finish_loading do + has_element?(:selected_label_content, label_name: label) + end + end + + def has_no_label?(label) + wait_labels_block_finish_loading do + has_no_element?(:selected_label_content, label_name: label) end end @@ -103,33 +105,34 @@ module QA find_element(:more_assignees_link) end - def select_labels_and_refresh(labels) - Support::Retrier.retry_until do - click_element(:labels_edit_button) - has_element?(:labels_dropdown_content, text: labels.first) - end + def select_labels(labels) + within_element(:labels_block) do + click_element(:edit_link) - labels.each do |label| - within_element(:labels_dropdown_content) do - send_keys_to_element(:dropdown_input_field, [label, :enter]) + labels.each do |label| + within_element(:labels_dropdown_content) do + fill_element(:dropdown_input_field, label) + click_button(text: label) + end end end - click_element(:labels_edit_button) - - labels.each do |label| - has_element?(:labels_block, text: label, wait: 0) - end - - refresh - - wait_for_requests + click_element(:title) # to blur dropdown end def toggle_more_assignees_link click_element(:more_assignees_link) end + # When the labels_widget feature flag is enabled, wait until the labels widget appears + def wait_for_labels_widget_feature_flag + Support::Retrier.retry_until(max_duration: 60, reload_page: page, retry_on_exception: true, sleep_interval: 5) do + within_element(:labels_block) do + find_element(:edit_link) + end + end + end + private def wait_assignees_block_finish_loading @@ -141,6 +144,15 @@ module QA end end + def wait_labels_block_finish_loading + within_element(:labels_block) do + wait_until(reload: false, max_duration: 10, sleep_interval: 1) do + finished_loading_block? + yield + end + end + end + def wait_milestone_block_finish_loading within_element(:milestone_block) do wait_until(reload: false, max_duration: 10, sleep_interval: 1) do diff --git a/qa/qa/page/component/issue_board/show.rb b/qa/qa/page/component/issue_board/show.rb index 1c1f7ab17f3..2259a65b546 100644 --- a/qa/qa/page/component/issue_board/show.rb +++ b/qa/qa/page/component/issue_board/show.rb @@ -24,14 +24,6 @@ module QA element :create_new_board_button end - view 'app/assets/javascripts/vue_shared/components/sidebar/labels_select_vue/dropdown_contents.vue' do - element :labels_dropdown_content - end - - view 'app/assets/javascripts/vue_shared/components/sidebar/labels_select_vue/dropdown_title.vue' do - element :labels_edit_button - end - view 'app/assets/javascripts/boards/components/board_content.vue' do element :boards_list end @@ -85,6 +77,7 @@ module QA def click_boards_config_button click_element(:boards_config_button) + wait_for_requests end def click_boards_dropdown_button @@ -97,16 +90,6 @@ module QA click_element(:focus_mode_button) end - def configure_by_label(label) - click_boards_config_button - click_element(:labels_edit_button) - find_element(:labels_dropdown_content).find('li', text: label).click - # Clicking the edit button again closes the dropdown and allows the save button to be clicked - click_element(:labels_edit_button) - click_element(:save_changes_button) - wait_boards_list_finish_loading - end - def create_new_board(board_name) click_boards_dropdown_button click_element(:create_new_board_button) diff --git a/qa/qa/page/group/bulk_import.rb b/qa/qa/page/group/bulk_import.rb index a62823f3469..90bc7a66dcc 100644 --- a/qa/qa/page/group/bulk_import.rb +++ b/qa/qa/page/group/bulk_import.rb @@ -34,7 +34,7 @@ module QA click_element(:target_namespace_selector_dropdown) click_element(:target_group_dropdown_item, group_name: target_group_name) - retry_until do + retry_until(message: "Triggering import") do click_element(:import_group_button) # Make sure import started before waiting for completion has_no_element?(:import_status_indicator, text: "Not started", wait: 1) diff --git a/qa/qa/page/group/settings/package_registries.rb b/qa/qa/page/group/settings/package_registries.rb index 5c93c0d6222..433872a378a 100644 --- a/qa/qa/page/group/settings/package_registries.rb +++ b/qa/qa/page/group/settings/package_registries.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true - module QA module Page module Group @@ -20,22 +19,33 @@ module QA def set_allow_duplicates_disabled expand_content :package_registry_settings_content do - click_element(:allow_duplicates_toggle) if duplicates_enabled? + click_on_allow_duplicates_button if duplicates_enabled? end end def set_allow_duplicates_enabled expand_content :package_registry_settings_content do - click_element(:allow_duplicates_toggle) if duplicates_disabled? + click_on_allow_duplicates_button unless duplicates_enabled? + end + end + + def click_on_allow_duplicates_button + with_allow_duplicates_button do |button| + button.click end end def duplicates_enabled? - has_element?(:allow_duplicates_label, text: 'Allow duplicates') + with_allow_duplicates_button do |button| + button[:class].include?('is-checked') + end end - def duplicates_disabled? - has_element?(:allow_duplicates_label, text: 'Do not allow duplicates') + def with_allow_duplicates_button + within_element :allow_duplicates_toggle do + toggle = find('button.gl-toggle:not(.is-disabled)') + yield(toggle) + end end def has_dependency_proxy_enabled? diff --git a/qa/qa/page/group/show.rb b/qa/qa/page/group/show.rb index 38d919be4db..2cd78f9f17a 100644 --- a/qa/qa/page/group/show.rb +++ b/qa/qa/page/group/show.rb @@ -9,6 +9,7 @@ module QA view 'app/views/groups/_home_panel.html.haml' do element :new_project_button element :new_subgroup_button + element :group_id_content end view 'app/assets/javascripts/groups/constants.js' do @@ -40,6 +41,10 @@ module QA click_element :new_project_button end + def group_id + find_element(:group_id_content).text.delete('Group ID: ') + end + def leave_group accept_alert do click_element :leave_group_link diff --git a/qa/qa/page/main/login.rb b/qa/qa/page/main/login.rb index c3170478733..5cba9d4bce4 100644 --- a/qa/qa/page/main/login.rb +++ b/qa/qa/page/main/login.rb @@ -45,6 +45,10 @@ module QA has_element?(:sign_in_button) end + def on_login_page? + has_element?(:login_page, wait: 0) + end + def sign_in_using_credentials(user: nil, skip_page_validation: false) # Don't try to log-in if we're already logged-in return if Page::Main::Menu.perform(&:signed_in?) @@ -164,6 +168,8 @@ module QA fill_element :password_field, user.password click_element :sign_in_button + Support::WaitForRequests.wait_for_requests + Page::Main::Terms.perform do |terms| terms.accept_terms if terms.visible? end diff --git a/qa/qa/page/main/menu.rb b/qa/qa/page/main/menu.rb index ad5cd971afc..e3bb585955b 100644 --- a/qa/qa/page/main/menu.rb +++ b/qa/qa/page/main/menu.rb @@ -4,6 +4,8 @@ module QA module Page module Main class Menu < Page::Base + prepend Mobile::Page::Main::Menu if Runtime::Env.mobile_layout? + view 'app/views/layouts/header/_current_user_dropdown.html.haml' do element :sign_out_link element :edit_profile_link @@ -12,12 +14,12 @@ module QA view 'app/views/layouts/header/_default.html.haml' do element :navbar, required: true element :canary_badge_link - element :user_avatar, required: true - element :user_menu, required: true + element :user_avatar, required: !QA::Runtime::Env.mobile_layout? + element :user_menu, required: !QA::Runtime::Env.mobile_layout? element :stop_impersonation_link - element :issues_shortcut_button, required: true - element :merge_requests_shortcut_button, required: true - element :todos_shortcut_button, required: true + element :issues_shortcut_button, required: !QA::Runtime::Env.mobile_layout? + element :merge_requests_shortcut_button, required: !QA::Runtime::Env.mobile_layout? + element :todos_shortcut_button, required: !QA::Runtime::Env.mobile_layout? end view 'app/assets/javascripts/nav/components/top_nav_app.vue' do @@ -98,10 +100,14 @@ module QA end def signed_in? + return false if Page::Main::Login.perform(&:on_login_page?) + has_personal_area?(wait: 0) end def not_signed_in? + return true if Page::Main::Login.perform(&:on_login_page?) + has_no_personal_area? end @@ -115,7 +121,7 @@ module QA click_element :sign_out_link end - has_no_element?(:user_avatar) + not_signed_in? end end diff --git a/qa/qa/page/merge_request/show.rb b/qa/qa/page/merge_request/show.rb index d4fa3b38f02..f8d063ac6bd 100644 --- a/qa/qa/page/merge_request/show.rb +++ b/qa/qa/page/merge_request/show.rb @@ -99,6 +99,8 @@ module QA view 'app/assets/javascripts/vue_shared/components/markdown/suggestion_diff_header.vue' do element :add_suggestion_batch_button + element :applied_badge + element :applying_badge end view 'app/views/projects/merge_requests/_description.html.haml' do @@ -354,7 +356,7 @@ module QA end def apply_suggestion_with_message(message) - click_element(:apply_suggestion_dropdown) + all_elements(:apply_suggestion_dropdown, minimum: 1).first.click fill_element(:commit_message_field, message) click_element(:commit_with_custom_message_button) end @@ -363,6 +365,13 @@ module QA all_elements(:add_suggestion_batch_button, minimum: 1).first.click end + def has_suggestions_applied?(count = 1) + wait_until(reload: false) do + has_no_element?(:applying_badge) + end + all_elements(:applied_badge, count: count) + end + def cherry_pick! click_element(:cherry_pick_button, Page::Component::CommitModal) click_element(:submit_commit_button) diff --git a/qa/qa/page/profile/menu.rb b/qa/qa/page/profile/menu.rb index a12db2918dc..d638a378610 100644 --- a/qa/qa/page/profile/menu.rb +++ b/qa/qa/page/profile/menu.rb @@ -4,6 +4,10 @@ module QA module Page module Profile class Menu < Page::Base + # We need to check remote_mobile_device_name instead of mobile_layout? here + # since tablets have the regular top navigation bar but still close the left nav + prepend QA::Mobile::Page::Profile::Menu if QA::Runtime::Env.remote_mobile_device_name + view 'app/views/layouts/nav/sidebar/_profile.html.haml' do element :access_token_link, 'link_to profile_personal_access_tokens_path' # rubocop:disable QA/ElementWithPattern element :access_token_title, 'Access Tokens' # rubocop:disable QA/ElementWithPattern diff --git a/qa/qa/page/project/fork/new.rb b/qa/qa/page/project/fork/new.rb index 7062702679a..cd743b648d8 100644 --- a/qa/qa/page/project/fork/new.rb +++ b/qa/qa/page/project/fork/new.rb @@ -12,6 +12,7 @@ module QA view 'app/assets/javascripts/pages/projects/forks/new/components/fork_form.vue' do element :fork_namespace_dropdown element :fork_project_button + element :fork_privacy_button end def fork_project(namespace = Runtime::Namespace.path) @@ -19,6 +20,7 @@ module QA click_element(:fork_namespace_button, name: namespace) else select_element(:fork_namespace_dropdown, namespace) + click_element(:fork_privacy_button, privacy_level: 'public') click_element(:fork_project_button) end end diff --git a/qa/qa/page/project/import/github.rb b/qa/qa/page/project/import/github.rb index bb35c5eb17c..47f7e701ae8 100644 --- a/qa/qa/page/project/import/github.rb +++ b/qa/qa/page/project/import/github.rb @@ -49,7 +49,12 @@ module QA click_element(:target_namespace_selector_dropdown) click_element(:target_group_dropdown_item, group_name: target_group_path) fill_element(:project_path_field, project_name) - click_element(:import_button) + + retry_until do + click_element(:import_button) + # Make sure import started before waiting for completion + has_no_element?(:import_status_indicator, text: "Not started", wait: 1) + end end end diff --git a/qa/qa/page/project/infrastructure/kubernetes/add.rb b/qa/qa/page/project/infrastructure/kubernetes/add.rb index e2d50c1bcf1..ed9ecb51a46 100644 --- a/qa/qa/page/project/infrastructure/kubernetes/add.rb +++ b/qa/qa/page/project/infrastructure/kubernetes/add.rb @@ -11,7 +11,7 @@ module QA end def add_existing_cluster - click_element(:add_existing_cluster_tab) + page.find('.gl-tab-nav-item', text: 'Connect existing cluster').click end end end diff --git a/qa/qa/page/project/infrastructure/kubernetes/index.rb b/qa/qa/page/project/infrastructure/kubernetes/index.rb index bdcaf7ffaff..0424682179e 100644 --- a/qa/qa/page/project/infrastructure/kubernetes/index.rb +++ b/qa/qa/page/project/infrastructure/kubernetes/index.rb @@ -6,12 +6,12 @@ module QA module Infrastructure module Kubernetes class Index < Page::Base - view 'app/views/clusters/clusters/_empty_state.html.haml' do - element :add_kubernetes_cluster_link + view 'app/assets/javascripts/clusters_list/components/clusters_view_all.vue' do + element :connect_existing_cluster_button end - def add_kubernetes_cluster - click_element :add_kubernetes_cluster_link + def connect_existing_cluster + click_link 'Connect existing cluster' end def has_cluster?(cluster) diff --git a/qa/qa/page/project/infrastructure/kubernetes/show.rb b/qa/qa/page/project/infrastructure/kubernetes/show.rb index 62a04a53a2f..6de5024e525 100644 --- a/qa/qa/page/project/infrastructure/kubernetes/show.rb +++ b/qa/qa/page/project/infrastructure/kubernetes/show.rb @@ -7,26 +7,9 @@ module QA module Kubernetes class Show < Page::Base view 'app/assets/javascripts/clusters/forms/components/integration_form.vue' do - element :integration_status_toggle, required: true - element :base_domain_field, required: true - element :save_changes_button, required: true - end - - view 'app/views/clusters/clusters/_details_tab.html.haml' do - element :details, required: true - end - - view 'app/views/clusters/clusters/_health.html.haml' do - element :cluster_health_section - end - - view 'app/views/clusters/clusters/_health_tab.html.haml' do - element :health, required: true - end - - def open_details - has_element?(:details, wait: 30) - click_element :details + element :integration_status_toggle + element :base_domain_field + element :save_changes_button end def set_domain(domain) @@ -36,29 +19,6 @@ module QA def save_domain click_element :save_changes_button, Page::Project::Infrastructure::Kubernetes::Show end - - def wait_for_cluster_health - wait_until(max_duration: 120, sleep_interval: 3, reload: true) do - has_cluster_health_graphs? - end - end - - def open_health - has_element?(:health, wait: 30) - click_element :health - end - - def has_cluster_health_graphs? - within_cluster_health_section do - has_text?('CPU Usage') - end - end - - def within_cluster_health_section - within_element :cluster_health_section do - yield - end - end end end end diff --git a/qa/qa/page/project/issue/show.rb b/qa/qa/page/project/issue/show.rb index 7a5a153db86..3b033830420 100644 --- a/qa/qa/page/project/issue/show.rb +++ b/qa/qa/page/project/issue/show.rb @@ -9,6 +9,7 @@ module QA include Page::Component::Note include Page::Component::DesignManagement include Page::Component::Issuable::Sidebar + prepend Mobile::Page::Project::Issue::Show if Runtime::Env.mobile_layout? view 'app/assets/javascripts/vue_shared/components/issue/related_issuable_item.vue' do element :remove_related_issue_button @@ -64,6 +65,10 @@ module QA def has_metrics_unfurled? has_element?(:prometheus_graph_widgets, wait: 30) end + + def has_reopen_issue_button? + has_element?(:reopen_issue_button) + end end end end diff --git a/qa/qa/page/project/job/show.rb b/qa/qa/page/project/job/show.rb index 78b6bebe02e..2fb925b3930 100644 --- a/qa/qa/page/project/job/show.rb +++ b/qa/qa/page/project/job/show.rb @@ -62,6 +62,12 @@ module QA has_element? :job_log_content end + def has_status?(status, wait: 30) + wait_until(reload: false, max_duration: wait, sleep_interval: 1) do + status_badge == status + end + end + private def loaded?(wait: 60) diff --git a/qa/qa/page/project/monitor/metrics/show.rb b/qa/qa/page/project/monitor/metrics/show.rb index 0129ee06cb6..70ebc795595 100644 --- a/qa/qa/page/project/monitor/metrics/show.rb +++ b/qa/qa/page/project/monitor/metrics/show.rb @@ -31,7 +31,6 @@ module QA view 'app/assets/javascripts/monitoring/components/dashboard_panel.vue' do element :prometheus_graph_widgets element :prometheus_widgets_dropdown - element :alert_widget_menu_item element :generate_chart_link_menu_item end diff --git a/qa/qa/page/project/new.rb b/qa/qa/page/project/new.rb index 3ecdabeeed2..5ff52527774 100644 --- a/qa/qa/page/project/new.rb +++ b/qa/qa/page/project/new.rb @@ -13,6 +13,7 @@ module QA view 'app/views/projects/_new_project_fields.html.haml' do element :initialize_with_readme_checkbox + element :initialize_with_sast_checkbox element :project_namespace_field, 'namespaces_options' # rubocop:disable QA/ElementWithPattern element :project_name, 'text_field :name' # rubocop:disable QA/ElementWithPattern element :project_path, 'text_field :path' # rubocop:disable QA/ElementWithPattern @@ -79,6 +80,13 @@ module QA choose visibility.capitalize end + # Disable experiment for SAST at project creation https://gitlab.com/gitlab-org/gitlab/-/issues/333196 + def disable_initialize_with_sast + return unless has_element?(:initialize_with_sast_checkbox) + + uncheck_element(:initialize_with_sast_checkbox) + end + def click_github_link click_link 'GitHub' end diff --git a/qa/qa/page/project/registry/show.rb b/qa/qa/page/project/registry/show.rb index f2472a83401..270445560be 100644 --- a/qa/qa/page/project/registry/show.rb +++ b/qa/qa/page/project/registry/show.rb @@ -5,15 +5,15 @@ module QA module Project module Registry class Show < QA::Page::Base - view 'app/assets/javascripts/registry/explorer/components/list_page/image_list_row.vue' do + view 'app/assets/javascripts/packages_and_registries/container_registry/explorer/components/list_page/image_list_row.vue' do element :registry_image_content end - view 'app/assets/javascripts/registry/explorer/components/details_page/tags_list_row.vue' do + view 'app/assets/javascripts/packages_and_registries/container_registry/explorer/components/details_page/tags_list_row.vue' do element :more_actions_menu end - view 'app/assets/javascripts/registry/explorer/components/details_page/tags_list_row.vue' do + view 'app/assets/javascripts/packages_and_registries/container_registry/explorer/components/details_page/tags_list_row.vue' do element :tag_delete_button end diff --git a/qa/qa/page/project/settings/deploy_tokens.rb b/qa/qa/page/project/settings/deploy_tokens.rb index 7b61c81154a..407d57bc54e 100644 --- a/qa/qa/page/project/settings/deploy_tokens.rb +++ b/qa/qa/page/project/settings/deploy_tokens.rb @@ -12,6 +12,7 @@ module QA element :deploy_token_read_package_registry_checkbox element :deploy_token_write_package_registry_checkbox element :deploy_token_read_registry_checkbox + element :deploy_token_write_registry_checkbox element :create_deploy_token_button end @@ -29,11 +30,12 @@ module QA fill_element(:deploy_token_expires_at_field, expires_at.to_s + "\n") end - def fill_scopes(read_repository: false, read_registry: false, read_package_registry: false, write_package_registry: false) - check_element(:deploy_token_read_repository_checkbox) if read_repository - check_element(:deploy_token_read_package_registry_checkbox) if read_package_registry - check_element(:deploy_token_write_package_registry_checkbox) if write_package_registry - check_element(:deploy_token_read_registry_checkbox) if read_registry + def fill_scopes(scopes) + check_element(:deploy_token_read_repository_checkbox) if scopes.include? :read_repository + check_element(:deploy_token_read_package_registry_checkbox) if scopes.include? :read_package_registry + check_element(:deploy_token_write_package_registry_checkbox) if scopes.include? :write_package_registry + check_element(:deploy_token_read_registry_checkbox) if scopes.include? :read_registry + check_element(:deploy_token_write_registry_checkbox) if scopes.include? :write_registry end def add_token diff --git a/qa/qa/page/project/show.rb b/qa/qa/page/project/show.rb index 6e5097c3812..65a1f726a8a 100644 --- a/qa/qa/page/project/show.rb +++ b/qa/qa/page/project/show.rb @@ -9,6 +9,7 @@ module QA include Page::Component::Breadcrumbs include Page::Project::SubMenus::Settings include Page::File::Shared::CommitMessage + prepend Mobile::Page::Project::Show if Runtime::Env.mobile_layout? view 'app/assets/javascripts/repository/components/preview/index.vue' do element :blob_viewer_content @@ -117,7 +118,7 @@ module QA end def go_to_new_issue - click_element :new_menu_toggle + click_element(:new_menu_toggle) click_element(:new_issue_link) end @@ -153,6 +154,10 @@ module QA click_element(:web_ide_button) end + def open_web_ide_via_shortcut + page.driver.send_keys('.') + end + def has_edit_fork_button? has_element?(:web_ide_button, text: 'Edit fork in Web IDE') end diff --git a/qa/qa/page/project/sub_menus/common.rb b/qa/qa/page/project/sub_menus/common.rb index c20710bc393..112f49a90ee 100644 --- a/qa/qa/page/project/sub_menus/common.rb +++ b/qa/qa/page/project/sub_menus/common.rb @@ -19,6 +19,10 @@ module QA view 'app/views/shared/nav/_sidebar_menu.html.haml' do element :sidebar_menu_link end + + view 'app/views/layouts/nav/_breadcrumbs.html.haml' do + element :toggle_mobile_nav_button + end end end diff --git a/qa/qa/page/sub_menus/common.rb b/qa/qa/page/sub_menus/common.rb index 2efeeb062e8..518b3b4e84e 100644 --- a/qa/qa/page/sub_menus/common.rb +++ b/qa/qa/page/sub_menus/common.rb @@ -4,6 +4,10 @@ module QA module Page module SubMenus module Common + # We need to check remote_mobile_device_name instead of mobile_layout? here + # since tablets have the regular top navigation bar but still close the left nav + prepend Mobile::Page::SubMenus::Common if QA::Runtime::Env.remote_mobile_device_name + def hover_element(element) within_sidebar do find_element(element).hover diff --git a/qa/qa/resource/base.rb b/qa/qa/resource/base.rb index a7243b7ebc2..26a2a668cc1 100644 --- a/qa/qa/resource/base.rb +++ b/qa/qa/resource/base.rb @@ -77,17 +77,24 @@ module QA def log_fabrication(method, resource, parents, args) start = Time.now - yield.tap do + Support::FabricationTracker.start_fabrication + result = yield.tap do + fabrication_time = Time.now - start + + Support::FabricationTracker.save_fabrication(:"#{method}_fabrication", fabrication_time) Runtime::Logger.debug do msg = ["==#{'=' * parents.size}>"] msg << "Built a #{name}" msg << "as a dependency of #{parents.last}" if parents.any? msg << "via #{method}" - msg << "in #{Time.now - start} seconds" + msg << "in #{fabrication_time} seconds" msg.join(' ') end end + Support::FabricationTracker.finish_fabrication + + result end # Define custom attribute diff --git a/qa/qa/resource/bulk_import_group.rb b/qa/qa/resource/bulk_import_group.rb index 5380bb16f10..e8dc2d291b8 100644 --- a/qa/qa/resource/bulk_import_group.rb +++ b/qa/qa/resource/bulk_import_group.rb @@ -59,6 +59,9 @@ module QA } end + # Get import status + # + # @return [String] def import_status response = get(Runtime::API::Request.new(api_client, "/bulk_imports/#{import_id}").url) @@ -69,6 +72,15 @@ module QA parse_body(response)[:status] end + # Get import details + # + # @return [Array] + def import_details + response = get(Runtime::API::Request.new(api_client, "/bulk_imports/#{import_id}/entities").url) + + parse_body(response) + end + private def transform_api_resource(api_resource) diff --git a/qa/qa/resource/clusters/agent.rb b/qa/qa/resource/clusters/agent.rb index cad5a4c6b1d..ee5a292b9b3 100644 --- a/qa/qa/resource/clusters/agent.rb +++ b/qa/qa/resource/clusters/agent.rb @@ -19,13 +19,12 @@ module QA def fabricate! puts 'TODO: FABRICATE VIA UI' end - # TODO - # - # The UI for this model is not yet implemented. So far it can only be - # created through the GraphQL API - # def fabricate - # - # end + + def resource_web_url(resource) + super + rescue ResourceURLMissingError + # this particular resource does not expose a web_url property + end def api_get_path "gid://gitlab/Clusters::Agent/#{id}" diff --git a/qa/qa/resource/clusters/agent_token.rb b/qa/qa/resource/clusters/agent_token.rb index 3286f46cdb2..6d803b94564 100644 --- a/qa/qa/resource/clusters/agent_token.rb +++ b/qa/qa/resource/clusters/agent_token.rb @@ -13,13 +13,12 @@ module QA def fabricate! puts 'TODO: FABRICATE VIA UI' end - # TODO - # - # The UI for this model is not yet implemented. So far it can only be - # created through the GraphQL API - # def fabricate - # - # end + + def resource_web_url(resource) + super + rescue ResourceURLMissingError + # this particular resource does not expose a web_url property + end def api_get_path "gid://gitlab/Clusters::AgentToken/#{id}" diff --git a/qa/qa/resource/deploy_token.rb b/qa/qa/resource/deploy_token.rb index 151454c37b1..f5d3b87fc2b 100644 --- a/qa/qa/resource/deploy_token.rb +++ b/qa/qa/resource/deploy_token.rb @@ -4,6 +4,7 @@ module QA module Resource class DeployToken < Base attr_accessor :name, :expires_at + attr_writer :scopes attribute :username do Page::Project::Settings::Repository.perform do |repository_page| @@ -37,7 +38,7 @@ module QA setting.expand_deploy_tokens do |page| page.fill_token_name(name) page.fill_token_expires_at(expires_at) - page.fill_scopes(read_repository: true, read_package_registry: true, write_package_registry: true) + page.fill_scopes(@scopes) page.add_token end diff --git a/qa/qa/resource/file.rb b/qa/qa/resource/file.rb index 4ca180373f6..9b05c0cb456 100644 --- a/qa/qa/resource/file.rb +++ b/qa/qa/resource/file.rb @@ -67,7 +67,7 @@ module QA private def transform_api_resource(api_resource) - api_resource[:web_url] = "#{Runtime::Scenario.gitlab_address}/#{project.full_path}/-/tree/#{branch}/#{api_resource[:file_path]}" + api_resource[:web_url] = "#{Runtime::Scenario.gitlab_address}/#{project.full_path}/-/blob/#{branch}/#{api_resource[:file_path]}" api_resource end end diff --git a/qa/qa/resource/fork.rb b/qa/qa/resource/fork.rb index b3814011f2c..d0313670e8b 100644 --- a/qa/qa/resource/fork.rb +++ b/qa/qa/resource/fork.rb @@ -37,11 +37,9 @@ module QA namespace_path ||= user.name # Sign out as admin and sign is as the fork user - Page::Main::Menu.perform(&:sign_out) - Runtime::Browser.visit(:gitlab, Page::Main::Login) - Page::Main::Login.perform do |login| - login.sign_in_using_credentials(user: user) - end + Flow::Login.sign_in(as: user) + + @api_client = Runtime::API::Client.new(:gitlab, is_new_session: false, user: user) upstream.visit! diff --git a/qa/qa/resource/issue.rb b/qa/qa/resource/issue.rb index 9214d4eff4a..344f177932f 100644 --- a/qa/qa/resource/issue.rb +++ b/qa/qa/resource/issue.rb @@ -9,6 +9,7 @@ module QA Project.fabricate! do |resource| resource.name = 'project-for-issues' resource.description = 'project for adding issues' + resource.api_client = api_client end end @@ -93,6 +94,52 @@ module QA attempts: attempts ) end + + # Object comparison + # + # @param [QA::Resource::Issue] other + # @return [Boolean] + def ==(other) + other.is_a?(Issue) && comparable_issue == other.comparable_issue + end + + # Override inspect for a better rspec failure diff output + # + # @return [String] + def inspect + JSON.pretty_generate(comparable_issue) + end + + protected + + # Return subset of fields for comparing issues + # + # @return [Hash] + def comparable_issue + reload! if api_response.nil? + + api_resource.slice( + :state, + :description, + :type, + :title, + :labels, + :milestone, + :upvotes, + :downvotes, + :merge_requests_count, + :user_notes_count, + :due_date, + :has_tasks, + :task_status, + :confidential, + :discussion_locked, + :issue_type, + :task_completion_status, + :closed_at, + :created_at + ) + end end end end diff --git a/qa/qa/resource/kubernetes_cluster/project_cluster.rb b/qa/qa/resource/kubernetes_cluster/project_cluster.rb index b3eba77fc46..0a63ff47694 100644 --- a/qa/qa/resource/kubernetes_cluster/project_cluster.rb +++ b/qa/qa/resource/kubernetes_cluster/project_cluster.rb @@ -13,8 +13,8 @@ module QA Resource::Project.fabricate! end - def ingress_ip - @ingress_ip ||= @cluster.fetch_external_ip_for_ingress + attribute :ingress_ip do + @cluster.fetch_external_ip_for_ingress end def fabricate! @@ -24,7 +24,7 @@ module QA &:go_to_infrastructure_kubernetes) Page::Project::Infrastructure::Kubernetes::Index.perform( - &:add_kubernetes_cluster) + &:connect_existing_cluster) Page::Project::Infrastructure::Kubernetes::Add.perform( &:add_existing_cluster) @@ -39,14 +39,10 @@ module QA end Page::Project::Infrastructure::Kubernetes::Show.perform do |show| - # We must wait a few seconds for permissions to be set up correctly for new cluster - sleep 25 - if @install_ingress - populate(:ingress_ip) + ingress_ip - show.open_details - show.set_domain("#{ingress_ip}.nip.io") + show.set_domain("#{@ingress_ip}.nip.io") show.save_domain end end diff --git a/qa/qa/resource/merge_request_from_fork.rb b/qa/qa/resource/merge_request_from_fork.rb index b0579cf37b8..512f3eb7bfc 100644 --- a/qa/qa/resource/merge_request_from_fork.rb +++ b/qa/qa/resource/merge_request_from_fork.rb @@ -6,7 +6,7 @@ module QA attr_accessor :fork_branch attribute :fork do - Fork.fabricate_via_api! + Fork.fabricate_via_browser_ui! end attribute :push do diff --git a/qa/qa/resource/project.rb b/qa/qa/resource/project.rb index 3f6a4eee5ac..864f3a14c3d 100644 --- a/qa/qa/resource/project.rb +++ b/qa/qa/resource/project.rb @@ -27,7 +27,9 @@ module QA :import_error attribute :group do - Group.fabricate! + Group.fabricate! do |group| + group.api_client = api_client + end end attribute :path_with_namespace do @@ -91,6 +93,7 @@ module QA new_page.choose_name(@name) new_page.add_description(@description) new_page.set_visibility(@visibility) + new_page.disable_initialize_with_sast new_page.disable_initialize_with_readme unless @initialize_with_readme new_page.create_new_project end @@ -214,6 +217,10 @@ module QA "#{api_get_path}/wikis" end + def api_push_rules_path + "#{api_get_path}/push_rule" + end + def api_post_body post_body = { name: name, @@ -358,6 +365,15 @@ module QA parse_body(response) end + def push_rules + response = get(request_url(api_push_rules_path)) + parse_body(response) + end + + def add_push_rules(rules) + api_post_to(api_push_rules_path, rules) + end + # Object comparison # # @param [QA::Resource::Project] other diff --git a/qa/qa/resource/sandbox.rb b/qa/qa/resource/sandbox.rb index b351d78a184..385e0fa4f7e 100644 --- a/qa/qa/resource/sandbox.rb +++ b/qa/qa/resource/sandbox.rb @@ -7,6 +7,17 @@ module QA # creating it if it doesn't yet exist. # class Sandbox < GroupBase + class << self + # Force top level group creation via UI if test is executed on dot_com environment + def fabricate!(*args, &prepare_block) + return fabricate_via_browser_ui!(*args, &prepare_block) if Specs::Helpers::ContextSelector.dot_com? + + fabricate_via_api!(*args, &prepare_block) + rescue NotImplementedError + fabricate_via_browser_ui!(*args, &prepare_block) + end + end + def initialize @path = Runtime::Namespace.sandbox_name end @@ -14,6 +25,8 @@ module QA alias_method :full_path, :path def fabricate! + Flow::Login.sign_in_unless_signed_in + Page::Main::Menu.perform(&:go_to_groups) Page::Dashboard::Groups.perform do |groups_page| @@ -23,10 +36,13 @@ module QA groups_page.click_new_group Page::Group::New.perform do |group| + group.click_create_group group.set_path(path) group.set_visibility('Public') group.create end + + @id = Page::Group::Show.perform(&:group_id) end end end diff --git a/qa/qa/runtime/allure_report.rb b/qa/qa/runtime/allure_report.rb index 5f628050f3b..9ae04dbe111 100644 --- a/qa/qa/runtime/allure_report.rb +++ b/qa/qa/runtime/allure_report.rb @@ -76,6 +76,15 @@ module QA RSpec.configure do |config| config.add_formatter(AllureRspecFormatter) config.add_formatter(QA::Support::Formatters::AllureMetadataFormatter) + + config.append_after do |example| + Allure.add_attachment( + name: 'browser.log', + source: Capybara.current_session.driver.browser.logs.get(:browser).map(&:to_s).join("\n\n"), + type: Allure::ContentType::TXT, + test_case: true + ) + end end end diff --git a/qa/qa/runtime/browser.rb b/qa/qa/runtime/browser.rb index 0566bc237bb..f1d93ce376a 100644 --- a/qa/qa/runtime/browser.rb +++ b/qa/qa/runtime/browser.rb @@ -99,7 +99,7 @@ module QA end # Disable /dev/shm use in CI. See https://gitlab.com/gitlab-org/gitlab/issues/4252 - capabilities['goog:chromeOptions'][:args] << 'disable-dev-shm-usage' if QA::Runtime::Env.running_in_ci? + capabilities['goog:chromeOptions'][:args] << 'disable-dev-shm-usage' if QA::Runtime::Env.disable_dev_shm? # Specify the user-agent to allow challenges to be bypassed # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/11938 @@ -205,6 +205,9 @@ module QA simulate_slow_connection if Runtime::Env.simulate_slow_connection? + # Wait until the new page is ready for us to interact with it + Support::WaitForRequests.wait_for_requests + page_class.validate_elements_present! if page_class.respond_to?(:validate_elements_present!) if QA::Runtime::Env.qa_cookies diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb index cdfa95457c7..163710a1510 100644 --- a/qa/qa/runtime/env.rb +++ b/qa/qa/runtime/env.rb @@ -80,6 +80,11 @@ module QA enabled?(ENV['CHROME_REUSE_PROFILE'], default: false) end + # Disable /dev/shm use in CI. See https://gitlab.com/gitlab-org/gitlab/issues/4252 + def disable_dev_shm? + running_in_ci? || enabled?(ENV['CHROME_DISABLE_DEV_SHM'], default: false) + end + def accept_insecure_certs? enabled?(ENV['ACCEPT_INSECURE_CERTS']) end @@ -153,6 +158,12 @@ module QA ENV['QA_REMOTE_MOBILE_DEVICE_NAME'] end + def mobile_layout? + return false if ENV['QA_REMOTE_MOBILE_DEVICE_NAME'].blank? + + !(ENV['QA_REMOTE_MOBILE_DEVICE_NAME'].downcase.include?('ipad') || ENV['QA_REMOTE_MOBILE_DEVICE_NAME'].downcase.include?('tablet')) + end + def user_username ENV['GITLAB_USERNAME'] end @@ -392,7 +403,7 @@ module QA end def gitlab_agentk_version - ENV.fetch('GITLAB_AGENTK_VERSION', 'v13.7.0') + ENV.fetch('GITLAB_AGENTK_VERSION', 'v14.4.0') end def transient_trials diff --git a/qa/qa/runtime/feature.rb b/qa/qa/runtime/feature.rb index 58408524f54..ec28813c1f6 100644 --- a/qa/qa/runtime/feature.rb +++ b/qa/qa/runtime/feature.rb @@ -5,15 +5,16 @@ require 'active_support/core_ext/object/blank' module QA module Runtime class Feature + SetFeatureError = Class.new(RuntimeError) + AuthorizationError = Class.new(RuntimeError) + UnknownScopeError = Class.new(RuntimeError) + UnknownStateError = Class.new(RuntimeError) + class << self # Documentation: https://docs.gitlab.com/ee/api/features.html include Support::API - SetFeatureError = Class.new(RuntimeError) - AuthorizationError = Class.new(RuntimeError) - UnknownScopeError = Class.new(RuntimeError) - def remove(key) request = Runtime::API::Request.new(api_client, "/features/#{key}") response = delete(request.url) @@ -30,6 +31,23 @@ module QA set_and_verify(key, enable: false, **scopes) end + # Set one or more flags to their specified state. + # + # @param [Hash] flags The feature flags and desired values, e.g., { 'flag1' => 'enabled', 'flag2' => "disabled" } + # @param [Hash] scopes The scope (user, project, group) to apply the feature flag to. + def set(flags, **scopes) + flags.each_pair do |flag, state| + case state + when 'enabled', 'enable', 'true', 1, true + enable(flag, **scopes) + when 'disabled', 'disable', 'false', 0, false + disable(flag, **scopes) + else + raise UnknownStateError, "Unknown feature flag state: #{state}" + end + end + end + def enabled?(key, **scopes) feature = JSON.parse(get_features).find { |flag| flag['name'] == key.to_s } feature && (feature['state'] == 'on' || feature['state'] == 'conditional' && scopes.present? && enabled_scope?(feature['gates'], **scopes)) @@ -47,15 +65,15 @@ module QA scopes.each do |key, value| case key when :project, :group, :user - actors = gates.filter { |i| i['key'] == 'actors' }.first['value'] - break actors.include?("#{key.to_s.capitalize}:#{value.id}") + actors = gates.find { |i| i['key'] == 'actors' }['value'] + return actors.include?("#{key.to_s.capitalize}:#{value.id}") when :feature_group - groups = gates.filter { |i| i['key'] == 'groups' }.first['value'] - break groups.include?(value) - else - raise UnknownScopeError, "Unknown scope: #{key}" + groups = gates.find { |i| i['key'] == 'groups' }['value'] + return groups.include?(value) end end + + raise UnknownScopeError, "Unknown scope in: #{scopes}" end def get_features diff --git a/qa/qa/scenario/bootable.rb b/qa/qa/scenario/bootable.rb index 841ac4dd560..ae180ffce1c 100644 --- a/qa/qa/scenario/bootable.rb +++ b/qa/qa/scenario/bootable.rb @@ -17,6 +17,22 @@ module QA arguments = OptionParser.new do |parser| options.to_a.each do |opt| + # The argument for the --set-feature-flags option should look something like "flag1=enabled,flag2=disabled" + # Here we translate that string into a hash, e.g.: { 'flag1' => 'enabled', 'flag2' => "disabled" } + if opt.name == :set_feature_flags + parser.on(opt.arg, opt.desc) do |flags| + value = flags.split(',').each_with_object({}) do |pair, hash| + flag_name, flag_value = pair.split('=') + raise '--set-feature-flags requires flag name and flag state for each flag, e.g., flag1=enabled,flag2=disabled' unless flag_name && flag_value + + hash[flag_name] = flag_value + end + Runtime::Scenario.define(opt.name, value) + end + + next + end + parser.on(opt.arg, opt.desc) do |value| Runtime::Scenario.define(opt.name, value) end diff --git a/qa/qa/scenario/shared_attributes.rb b/qa/qa/scenario/shared_attributes.rb index e2eaca42277..ddbe28f05d9 100644 --- a/qa/qa/scenario/shared_attributes.rb +++ b/qa/qa/scenario/shared_attributes.rb @@ -8,6 +8,9 @@ module QA attribute :gitlab_address, '--address URL', 'Address of the instance to test' attribute :enable_feature, '--enable-feature FEATURE_FLAG', 'Enable a feature before running tests' attribute :disable_feature, '--disable-feature FEATURE_FLAG', 'Disable a feature before running tests' + attribute :set_feature_flags, '--set-feature-flags FEATURE_FLAGS', + 'Set one or more feature flags before running tests. ' \ + 'Specify FEATURE_FLAGS as comma-separated flag=state pairs, e.g., "flag1=enabled,flag2=disabled"' attribute :parallel, '--parallel', 'Execute tests in parallel' attribute :loop, '--loop', 'Execute test repeatedly' end diff --git a/qa/qa/scenario/template.rb b/qa/qa/scenario/template.rb index d0a201e3d22..50bb952f1fd 100644 --- a/qa/qa/scenario/template.rb +++ b/qa/qa/scenario/template.rb @@ -38,8 +38,8 @@ module QA Runtime::Release.perform_before_hooks Runtime::Feature.enable(options[:enable_feature]) if options.key?(:enable_feature) - Runtime::Feature.disable(options[:disable_feature]) if options.key?(:disable_feature) && (@feature_enabled = Runtime::Feature.enabled?(options[:disable_feature])) + Runtime::Feature.set(options[:set_feature_flags]) if options.key?(:set_feature_flags) Specs::Runner.perform do |specs| specs.tty = true diff --git a/qa/qa/scenario/test/instance/reliable.rb b/qa/qa/scenario/test/instance/reliable.rb new file mode 100644 index 00000000000..725ab59f24a --- /dev/null +++ b/qa/qa/scenario/test/instance/reliable.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module QA + module Scenario + module Test + module Instance + class Reliable < Template + include Bootable + include SharedAttributes + + tags :reliable + end + end + end + end +end diff --git a/qa/qa/scenario/test/integration/ldap_no_tls.rb b/qa/qa/scenario/test/integration/ldap_no_tls.rb index bbf4c847f33..19f62b6ed80 100644 --- a/qa/qa/scenario/test/integration/ldap_no_tls.rb +++ b/qa/qa/scenario/test/integration/ldap_no_tls.rb @@ -1,3 +1,4 @@ +# rubocop:todo Naming/FileName # frozen_string_literal: true module QA @@ -11,3 +12,5 @@ module QA end end end + +# rubocop:enable Naming/FileName diff --git a/qa/qa/scenario/test/integration/ldap_tls.rb b/qa/qa/scenario/test/integration/ldap_tls.rb index 2a767e57bc6..109fbe6fd74 100644 --- a/qa/qa/scenario/test/integration/ldap_tls.rb +++ b/qa/qa/scenario/test/integration/ldap_tls.rb @@ -1,3 +1,4 @@ +# rubocop:todo Naming/FileName # frozen_string_literal: true module QA @@ -11,3 +12,5 @@ module QA end end end + +# rubocop:enable Naming/FileName diff --git a/qa/qa/scenario/test/integration/registry_tls.rb b/qa/qa/scenario/test/integration/registry_tls.rb new file mode 100644 index 00000000000..4e9d6b6ea97 --- /dev/null +++ b/qa/qa/scenario/test/integration/registry_tls.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module QA + module Scenario + module Test + module Integration + class RegistryTLS < Test::Instance::All + tags :registry_tls + end + end + end + end +end diff --git a/qa/qa/service/cluster_provider/gcloud.rb b/qa/qa/service/cluster_provider/gcloud.rb index f0fb5eee6e3..c6d1f6cfe88 100644 --- a/qa/qa/service/cluster_provider/gcloud.rb +++ b/qa/qa/service/cluster_provider/gcloud.rb @@ -24,16 +24,6 @@ module QA ) end - def set_credentials(admin_user) - master_auth = JSON.parse(`gcloud container clusters describe #{cluster_name} --region #{@region} --format 'json(masterAuth.username, masterAuth.password)'`) - - shell <<~CMD.tr("\n", ' ') - kubectl config set-credentials #{admin_user} - --username #{master_auth['masterAuth']['username']} - --password #{master_auth['masterAuth']['password']} - CMD - end - def setup login_if_not_already_logged_in create_cluster @@ -43,6 +33,12 @@ module QA delete_cluster end + def install_ingress + QA::Runtime::Logger.info "Attempting to install Ingress on cluster #{cluster_name}" + shell 'kubectl create -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-0.31.0/deploy/static/provider/cloud/deploy.yaml' + wait_for_ingress + end + private def login_if_not_already_logged_in @@ -59,7 +55,7 @@ module QA end def attempt_login_with_env_vars - puts "No gcloud account. Attempting to login from env vars GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY." + QA::Runtime::Logger.debug("Logging in with GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY.") gcloud_account_key = Tempfile.new('gcloud-account-key') gcloud_account_key.write(Runtime::Env.gcloud_account_key) gcloud_account_key.close @@ -80,7 +76,6 @@ module QA gcloud container clusters create #{cluster_name} #{auth_options} - --enable-basic-auth --region #{@region} --disk-size 10GB --num-nodes #{Runtime::Env.gcloud_num_nodes} @@ -109,6 +104,18 @@ module QA def get_region Runtime::Env.gcloud_region || @available_regions.delete(@available_regions.sample) end + + def wait_for_ingress + QA::Runtime::Logger.info 'Waiting for Ingress controller pod to be initialized' + + Support::Retrier.retry_until(max_attempts: 60, sleep_interval: 1) do + service_available?('kubectl get pods --all-namespaces -l app.kubernetes.io/component=controller | grep -o "ingress-nginx-controller.*1/1"') + end + end + + def service_available?(command) + system("#{command} > /dev/null 2>&1") + end end end end diff --git a/qa/qa/service/kubernetes_cluster.rb b/qa/qa/service/kubernetes_cluster.rb index 674bcdca9bb..ec53b9d8163 100644 --- a/qa/qa/service/kubernetes_cluster.rb +++ b/qa/qa/service/kubernetes_cluster.rb @@ -41,6 +41,10 @@ module QA cluster_name end + def install_ingress + @provider.install_ingress + end + def create_secret(secret, secret_name) shell("kubectl create secret generic #{secret_name} --from-literal=token='#{secret}'") end @@ -70,7 +74,13 @@ module QA end def fetch_external_ip_for_ingress - `kubectl get svc --all-namespaces --no-headers=true -l app.kubernetes.io/name=ingress-nginx -o custom-columns=:'status.loadBalancer.ingress[0].ip' | grep -v 'none'` + install_ingress + + # need to wait since the ingress-nginx service has an initial delay set of 10 seconds + sleep 10 + ingress_ip = `kubectl get svc --all-namespaces --no-headers=true -l app.kubernetes.io/name=ingress-nginx -o custom-columns=:'status.loadBalancer.ingress[0].ip' | grep -v 'none'` + QA::Runtime::Logger.debug "Has ingress address set to: #{ingress_ip}" + ingress_ip end private @@ -82,7 +92,6 @@ module QA def fetch_credentials return global_credentials unless rbac - @provider.set_credentials(admin_user) create_service_account(admin_user) account_credentials end diff --git a/qa/qa/service/praefect_manager.rb b/qa/qa/service/praefect_manager.rb index 71e3383a534..dbb49f18881 100644 --- a/qa/qa/service/praefect_manager.rb +++ b/qa/qa/service/praefect_manager.rb @@ -19,6 +19,8 @@ module QA @virtual_storage = 'default' end + attr_reader :primary_node, :secondary_node, :tertiary_node + # Executes the praefect `dataloss` command. # # @return [Boolean] whether dataloss has occurred @@ -376,7 +378,6 @@ module QA select job from replication_queue where state = 'ready' and job ->> 'change' = 'update' - and job ->> 'source_node_storage' = '#{current_primary_node}' and job ->> 'target_node_storage' = '#{@primary_node}'; SQL ) do |line| @@ -396,6 +397,97 @@ module QA result.size >= 5 end + def list_untracked_repositories + untracked_repositories = [] + shell "docker exec #{@praefect} bash -c 'gitlab-ctl praefect list-untracked-repositories'" do |line| + # Results look like this depending on whether untracked items found or not + # Running list-untracked-repositories + # Done. + + # Running list-untracked-repositories + # {"relative_path":"@hashed/aa/bb.git","storage":"gitaly1","virtual_storage":"default"} + # {"relative_path":"@hashed/bb/cc.git","storage":"gitaly3","virtual_storage":"default"} + # Done. + + QA::Runtime::Logger.debug(line.chomp) + next if line.start_with?('Running list-untracked-repositories') + next if line.start_with?('Done.') + + untracked_repositories.append(JSON.parse(line)) + end + + QA::Runtime::Logger.debug("list_untracked_repositories --- #{untracked_repositories}") + untracked_repositories + end + + def track_repository_in_praefect(relative_path, storage, virtual_storage) + cmd = "gitlab-ctl praefect track-repository --repository-relative-path #{relative_path} --authoritative-storage #{storage} --virtual-storage-name #{virtual_storage}" + shell "docker exec #{@praefect} bash -c '#{cmd}'" + end + + def remove_tracked_praefect_repository(relative_path, virtual_storage) + cmd = "gitlab-ctl praefect remove-repository --repository-relative-path #{relative_path} --virtual-storage-name #{virtual_storage}" + shell "docker exec #{@praefect} bash -c '#{cmd}'" + end + + def add_repo_to_disk(node, repo_path) + cmd = "GIT_DIR=. git init --initial-branch=main /var/opt/gitlab/git-data/repositories/#{repo_path}" + shell "docker exec --user git #{node} bash -c '#{cmd}'" + end + + def remove_repo_from_disk(repo_path) + cmd = "rm -rf /var/opt/gitlab/git-data/repositories/#{repo_path}" + shell "docker exec #{@primary_node} bash -c '#{cmd}'" + shell "docker exec #{@secondary_node} bash -c '#{cmd}'" + shell "docker exec #{@tertiary_node} bash -c '#{cmd}'" + end + + def remove_repository_from_praefect_database(relative_path) + shell sql_to_docker_exec_cmd("delete from repositories where relative_path = '#{relative_path}';") + shell sql_to_docker_exec_cmd("delete from storage_repositories where relative_path = '#{relative_path}';") + end + + def praefect_database_tracks_repo?(relative_path) + storage_repositories = [] + shell sql_to_docker_exec_cmd("SELECT count(*) FROM storage_repositories where relative_path='#{relative_path}';") do |line| + storage_repositories << line + end + QA::Runtime::Logger.debug("storage_repositories count is ---#{storage_repositories}") + + repositories = [] + shell sql_to_docker_exec_cmd("SELECT count(*) FROM repositories where relative_path='#{relative_path}';") do |line| + repositories << line + end + QA::Runtime::Logger.debug("repositories count is ---#{repositories}") + + (storage_repositories[2].to_i >= 1) && (repositories[2].to_i >= 1) + end + + def repository_replicated_to_disk?(node, relative_path) + Support::Waiter.wait_until(max_duration: 300, sleep_interval: 3, raise_on_failure: false) do + result = [] + shell sql_to_docker_exec_cmd("SELECT count(*) FROM storage_repositories where relative_path='#{relative_path}';") do |line| + result << line + end + QA::Runtime::Logger.debug("result is ---#{result}") + result[2].to_i == 3 + end + + repository_exists_on_node_disk?(node, relative_path) + end + + def repository_exists_on_node_disk?(node, relative_path) + # If the dir does not exist it has a non zero exit code leading to a error being raised + # Instead we echo a test line if the dir does not exist, which has a zero exit code, with no output + bash_command = "test -d /var/opt/gitlab/git-data/repositories/#{relative_path} || echo -n 'DIR_DOES_NOT_EXIST'" + result = [] + shell "docker exec #{node} bash -c '#{bash_command}'" do |line| + result << line + end + QA::Runtime::Logger.debug("result is ---#{result}") + result.exclude?("DIR_DOES_NOT_EXIST") + end + private def current_primary_node diff --git a/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb b/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb index 8b4900957c5..158881ed94c 100644 --- a/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb +++ b/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true module QA - RSpec.describe 'Manage', :requires_admin do + # run only base UI validation on staging because test requires top level group creation which is problematic + # on staging environment + RSpec.describe 'Manage', :requires_admin, except: { subdomain: :staging } do describe 'Bulk group import' do - let!(:staging?) { Runtime::Scenario.gitlab_address.include?('staging.gitlab.com') } - let(:import_wait_duration) { { max_duration: 300, sleep_interval: 2 } } let(:admin_api_client) { Runtime::API::Client.as_admin } let(:user) do @@ -38,11 +38,13 @@ module QA end before do - Runtime::Feature.enable(:top_level_group_creation_enabled) if staging? - sandbox.add_member(user, Resource::Members::AccessLevel::MAINTAINER) end + after do + user.remove_via_api! + end + context 'with subgroups and labels' do let(:subgroup) do Resource::Group.fabricate_via_api! do |group| @@ -155,12 +157,6 @@ module QA expect(imported_member.access_level).to eq(Resource::Members::AccessLevel::DEVELOPER) end end - - after do - user.remove_via_api! - ensure - Runtime::Feature.disable(:top_level_group_creation_enabled) if staging? - end end end end diff --git a/qa/qa/specs/features/api/1_manage/bulk_import_project_spec.rb b/qa/qa/specs/features/api/1_manage/bulk_import_project_spec.rb index 9935908d55e..14c94e99446 100644 --- a/qa/qa/specs/features/api/1_manage/bulk_import_project_spec.rb +++ b/qa/qa/specs/features/api/1_manage/bulk_import_project_spec.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true module QA - RSpec.describe 'Manage', :requires_admin do + # run only base UI validation on staging because test requires top level group creation which is problematic + # on staging environment + RSpec.describe 'Manage', :requires_admin, except: { subdomain: :staging } do describe 'Bulk project import' do - let!(:staging?) { Runtime::Scenario.gitlab_address.include?('staging.gitlab.com') } - let(:import_wait_duration) { { max_duration: 300, sleep_interval: 2 } } let(:admin_api_client) { Runtime::API::Client.as_admin } let(:user) do @@ -33,6 +33,7 @@ module QA Resource::Project.fabricate_via_api! do |project| project.api_client = api_client project.group = source_group + project.initialize_with_readme = true end end @@ -44,33 +45,87 @@ module QA end end + let(:imported_projects) do + imported_group.reload!.projects + end + + let(:project_import_failures) do + imported_group.import_details + .find { |entity| entity[:destination_name] == source_project.name } + &.fetch(:failures) + end + before do Runtime::Feature.enable(:bulk_import_projects) - Runtime::Feature.enable(:top_level_group_creation_enabled) if staging? sandbox.add_member(user, Resource::Members::AccessLevel::MAINTAINER) - source_project # fabricate source group and project + source_project.tap { |project| project.add_push_rules(member_check: true) } # fabricate source group and project end after do user.remove_via_api! ensure Runtime::Feature.disable(:bulk_import_projects) - Runtime::Feature.disable(:top_level_group_creation_enabled) if staging? end context 'with project' do + before do + imported_group # trigger import + end + it( 'successfully imports project', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/2297' ) do expect { imported_group.import_status }.to eventually_eq('finished').within(import_wait_duration) + expect(imported_projects.count).to eq(1), "Expected to have 1 imported project" - imported_projects = imported_group.reload!.projects aggregate_failures do - expect(imported_projects.count).to eq(1) expect(imported_projects.first).to eq(source_project) + expect(project_import_failures).to be_empty, "Expected no errors, was: #{project_import_failures}" + end + end + end + + context 'with project issues' do + let(:source_issue) do + Resource::Issue.fabricate_via_api! do |issue| + issue.api_client = api_client + issue.project = source_project + issue.labels = %w[label_one label_two] + end + end + + let(:imported_issues) do + imported_projects.first.issues + end + + let(:imported_issue) do + issue = imported_issues.first + Resource::Issue.init do |resource| + resource.api_client = api_client + resource.project = imported_projects.first + resource.iid = issue[:iid] + end + end + + before do + source_issue # fabricate source group, project, issue + imported_group # trigger import + end + + it( + 'successfully imports issue', + testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/2325' + ) do + expect { imported_group.import_status }.to eventually_eq('finished').within(import_wait_duration) + expect(imported_projects.count).to eq(1), "Expected to have 1 imported project" + + aggregate_failures do + expect(imported_issues.count).to eq(1) + expect(imported_issue.reload!).to eq(source_issue) + expect(project_import_failures).to be_empty, "Expected no errors, was: #{project_import_failures}" end end end diff --git a/qa/qa/specs/features/api/3_create/gitaly/automatic_failover_and_recovery_spec.rb b/qa/qa/specs/features/api/3_create/gitaly/automatic_failover_and_recovery_spec.rb index ec4f0387128..b85a0116f01 100644 --- a/qa/qa/specs/features/api/3_create/gitaly/automatic_failover_and_recovery_spec.rb +++ b/qa/qa/specs/features/api/3_create/gitaly/automatic_failover_and_recovery_spec.rb @@ -2,7 +2,7 @@ module QA RSpec.describe 'Create' do - context 'Gitaly automatic failover and recovery', :orchestrated, :gitaly_cluster, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/238953', type: :flaky } do + context 'Gitaly automatic failover and recovery', :orchestrated, :gitaly_cluster do # Variables shared between contexts. They're used and shared between # contexts so they can't be `let` variables. praefect_manager = Service::PraefectManager.new diff --git a/qa/qa/specs/features/api/3_create/gitaly/backend_node_recovery_spec.rb b/qa/qa/specs/features/api/3_create/gitaly/backend_node_recovery_spec.rb index 37670b70fd8..62437598f3b 100644 --- a/qa/qa/specs/features/api/3_create/gitaly/backend_node_recovery_spec.rb +++ b/qa/qa/specs/features/api/3_create/gitaly/backend_node_recovery_spec.rb @@ -3,7 +3,7 @@ module QA RSpec.describe 'Create' do context 'Gitaly' do - describe 'Backend node recovery', :orchestrated, :gitaly_cluster, :skip_live_env, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/322647', type: :flaky } do + describe 'Backend node recovery', :orchestrated, :gitaly_cluster, :skip_live_env do let(:praefect_manager) { Service::PraefectManager.new } let(:project) do Resource::Project.fabricate! do |project| @@ -31,14 +31,6 @@ module QA praefect_manager.stop_primary_node praefect_manager.wait_for_gitaly_check - # Confirm that we have access to the repo after failover - Support::Waiter.wait_until(retry_on_exception: true, sleep_interval: 5) do - Resource::Repository::Commit.fabricate_via_api! do |commits| - commits.project = project - commits.sha = project.default_branch - end - end - # Push a commit to the new primary Resource::Repository::ProjectPush.fabricate! do |push| push.project = project @@ -58,6 +50,11 @@ module QA # Wait for automatic replication praefect_manager.wait_for_replication(project.id) + # Force switch to the old primary node + # This ensures that the commit was replicated + praefect_manager.stop_secondary_node + praefect_manager.stop_tertiary_node + # Confirm that both commits are available expect(project.commits.map { |commit| commit[:message].chomp }) .to include("Initial commit").and include("pushed after failover") diff --git a/qa/qa/specs/features/api/3_create/gitaly/distributed_reads_spec.rb b/qa/qa/specs/features/api/3_create/gitaly/distributed_reads_spec.rb index 1aea1bd1189..dfc2de02bf0 100644 --- a/qa/qa/specs/features/api/3_create/gitaly/distributed_reads_spec.rb +++ b/qa/qa/specs/features/api/3_create/gitaly/distributed_reads_spec.rb @@ -16,9 +16,15 @@ module QA end before do + praefect_manager.start_all_nodes praefect_manager.wait_for_replication(project.id) end + after do + # Leave the cluster in a suitable state for subsequent tests + praefect_manager.start_all_nodes + end + it 'reads from each node', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1264' do pre_read_data = praefect_manager.query_read_distribution @@ -42,9 +48,7 @@ module QA after do # Leave the cluster in a suitable state for subsequent tests - praefect_manager.start_secondary_node - praefect_manager.wait_for_health_check_all_nodes - praefect_manager.wait_for_reliable_connection + praefect_manager.start_all_nodes end it 'does not read from the unhealthy node', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1263' do diff --git a/qa/qa/specs/features/api/3_create/gitaly/praefect_repo_sync_spec.rb b/qa/qa/specs/features/api/3_create/gitaly/praefect_repo_sync_spec.rb new file mode 100644 index 00000000000..07ea7971396 --- /dev/null +++ b/qa/qa/specs/features/api/3_create/gitaly/praefect_repo_sync_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Create' do + context 'Praefect repository commands', :orchestrated, :gitaly_cluster do + let(:praefect_manager) { Service::PraefectManager.new } + + let(:repo1) { { "relative_path" => "@hashed/repo1.git", "storage" => "gitaly1", "virtual_storage" => "default" } } + let(:repo2) { { "relative_path" => "@hashed/path/to/repo2.git", "storage" => "gitaly3", "virtual_storage" => "default" } } + + before do + praefect_manager.add_repo_to_disk(praefect_manager.primary_node, repo1["relative_path"]) + praefect_manager.add_repo_to_disk(praefect_manager.tertiary_node, repo2["relative_path"]) + end + + after do + praefect_manager.remove_repo_from_disk(repo1["relative_path"]) + praefect_manager.remove_repo_from_disk(repo2["relative_path"]) + praefect_manager.remove_repository_from_praefect_database(repo1["relative_path"]) + praefect_manager.remove_repository_from_praefect_database(repo2["relative_path"]) + end + + it 'allows admin to manage difference between praefect database and disk state', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/2344' do + # Some repos are on disk that praefect is not aware of + untracked_repositories = praefect_manager.list_untracked_repositories + expect(untracked_repositories).to include(repo1) + expect(untracked_repositories).to include(repo2) + + # admin manually adds the first repo to the praefect database + praefect_manager.track_repository_in_praefect(repo1["relative_path"], repo1["storage"], repo1["virtual_storage"]) + untracked_repositories = praefect_manager.list_untracked_repositories + expect(untracked_repositories).not_to include(repo1) + expect(untracked_repositories).to include(repo2) + expect(praefect_manager.repository_exists_on_node_disk?(praefect_manager.primary_node, repo1["relative_path"])).to be true + expect(praefect_manager.praefect_database_tracks_repo?(repo1["relative_path"])).to be true + + # admin manually adds the second repo to the praefect database + praefect_manager.track_repository_in_praefect(repo2["relative_path"], repo2["storage"], repo2["virtual_storage"]) + untracked_repositories = praefect_manager.list_untracked_repositories + expect(untracked_repositories).not_to include(repo2) + expect(praefect_manager.repository_exists_on_node_disk?(praefect_manager.tertiary_node, repo2["relative_path"])).to be true + expect(praefect_manager.praefect_database_tracks_repo?(repo2["relative_path"])).to be true + + # admin ensures replication to other nodes occurs + expect(praefect_manager.repository_replicated_to_disk?(praefect_manager.secondary_node, repo1["relative_path"])).to be true + expect(praefect_manager.repository_replicated_to_disk?(praefect_manager.tertiary_node, repo1["relative_path"])).to be true + expect(praefect_manager.repository_replicated_to_disk?(praefect_manager.primary_node, repo2["relative_path"])).to be true + expect(praefect_manager.repository_replicated_to_disk?(praefect_manager.secondary_node, repo2["relative_path"])).to be true + + # admin chooses to remove the first repo completely from praefect and disk + praefect_manager.remove_tracked_praefect_repository(repo1["relative_path"], repo1["virtual_storage"]) + expect(praefect_manager.repository_exists_on_node_disk?(praefect_manager.primary_node, repo1["relative_path"])).to be false + expect(praefect_manager.repository_exists_on_node_disk?(praefect_manager.secondary_node, repo1["relative_path"])).to be false + expect(praefect_manager.repository_exists_on_node_disk?(praefect_manager.tertiary_node, repo1["relative_path"])).to be false + expect(praefect_manager.praefect_database_tracks_repo?(repo1["relative_path"])).to be false + + untracked_repositories = praefect_manager.list_untracked_repositories + expect(untracked_repositories).not_to include(repo1) + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/1_manage/group/bulk_import_group_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/group/bulk_import_group_spec.rb index c136d14c1e5..15d51c14d26 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/group/bulk_import_group_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/group/bulk_import_group_spec.rb @@ -4,17 +4,16 @@ module QA RSpec.describe 'Manage', :requires_admin do describe 'Bulk group import' do let!(:staging?) { Runtime::Scenario.gitlab_address.include?('staging.gitlab.com') } - - let(:admin_api_client) { Runtime::API::Client.as_admin } - let(:user) do + let!(:admin_api_client) { Runtime::API::Client.as_admin } + let!(:user) do Resource::User.fabricate_via_api! do |usr| usr.api_client = admin_api_client usr.hard_delete_on_api_removal = true end end - let(:api_client) { Runtime::API::Client.new(user: user) } - let(:personal_access_token) { api_client.personal_access_token } + let!(:api_client) { Runtime::API::Client.new(user: user) } + let!(:personal_access_token) { api_client.personal_access_token } let(:sandbox) do Resource::Sandbox.fabricate_via_api! do |group| @@ -23,7 +22,7 @@ module QA end let(:source_group) do - Resource::Sandbox.fabricate_via_api! do |group| + Resource::Sandbox.fabricate! do |group| group.api_client = api_client group.path = "source-group-for-import-#{SecureRandom.hex(4)}" end @@ -38,14 +37,12 @@ module QA end before do - Runtime::Feature.enable(:top_level_group_creation_enabled) if staging? - sandbox.add_member(user, Resource::Members::AccessLevel::MAINTAINER) - # create groups explicitly before connecting gitlab instance + Flow::Login.sign_in(as: user) + source_group - Flow::Login.sign_in(as: user) Page::Main::Menu.perform(&:go_to_create_group) Page::Group::New.perform do |group| group.switch_to_import_tab @@ -53,6 +50,10 @@ module QA end end + after do + user.remove_via_api! + end + it( 'imports group from UI', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1806', @@ -72,12 +73,6 @@ module QA end end end - - after do - user.remove_via_api! - ensure - Runtime::Feature.disable(:top_level_group_creation_enabled) if staging? - end end end end diff --git a/qa/qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb index ca95d567316..9625771164c 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Manage', :smoke do + RSpec.describe 'Manage', :smoke, :mobile do describe 'basic user login' do it 'user logs in using basic credentials and logs out', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1578' do Flow::Login.sign_in diff --git a/qa/qa/specs/features/browser_ui/1_manage/login/maintain_log_in_mixed_env_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/login/maintain_log_in_mixed_env_spec.rb index 2b1c956039f..734529f319a 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/login/maintain_log_in_mixed_env_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/login/maintain_log_in_mixed_env_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Manage', :mixed_env, :smoke, only: { subdomain: :staging } do + RSpec.describe 'Manage', only: { subdomain: :staging }, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/344213', type: :stale } do describe 'basic user' do it 'remains logged in when redirected from canary to non-canary node', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/2251' do Runtime::Browser.visit(:gitlab, Page::Main::Login) diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb index af4e7126c29..34a7431e328 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb @@ -5,13 +5,13 @@ module QA describe 'Project', :requires_admin do shared_examples 'successful project creation' do it 'creates a new project' do - Page::Project::Show.perform do |project| - expect(project).to have_content(project_name) - expect(project).to have_content( + Page::Project::Show.perform do |project_page| + expect(project_page).to have_content(project_name) + expect(project_page).to have_content( /Project \S?#{project_name}\S+ was successfully created/ ) - expect(project).to have_content('create awesome project test') - expect(project).to have_content('The repository for this project is empty') + expect(project_page).to have_content('create awesome project test') + expect(project_page).to have_content('The repository for this project is empty') end end end diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/import_github_repo_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/import_github_repo_spec.rb index 4e3739a7672..c078c4bf12e 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/project/import_github_repo_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/project/import_github_repo_spec.rb @@ -4,7 +4,6 @@ module QA RSpec.describe 'Manage', :github, :requires_admin do describe 'Project import' do let(:github_repo) { 'gitlab-qa-github/test-project' } - let(:imported_project_name) { 'imported-project' } let(:api_client) { Runtime::API::Client.as_admin } let(:group) { Resource::Group.fabricate_via_api! { |resource| resource.api_client = api_client } } let(:user) do @@ -17,11 +16,10 @@ module QA let(:imported_project) do Resource::ProjectImportedFromGithub.init do |project| project.import = true - project.add_name_uuid = false - project.name = imported_project_name project.group = group project.github_personal_access_token = Runtime::Env.github_access_token project.github_repository_path = github_repo + project.api_client = api_client end end @@ -43,7 +41,7 @@ module QA it 'imports a GitHub repo', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1607' do Page::Project::Import::Github.perform do |import_page| import_page.add_personal_access_token(Runtime::Env.github_access_token) - import_page.import!(github_repo, group.full_path, imported_project_name) + import_page.import!(github_repo, group.full_path, imported_project.name) aggregate_failures do expect(import_page).to have_imported_project(github_repo) @@ -56,7 +54,7 @@ module QA imported_project.reload!.visit! Page::Project::Show.perform do |project| aggregate_failures do - expect(project).to have_content(imported_project_name) + expect(project).to have_content(imported_project.name) expect(project).to have_content('This test project is used for automated GitHub import by GitLab QA.') end end diff --git a/qa/qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb b/qa/qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb index 7519f4daae2..81ae8b82ef6 100644 --- a/qa/qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb +++ b/qa/qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb @@ -9,7 +9,7 @@ module QA Flow::Login.sign_in end - it 'creates an issue', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1185' do + it 'creates an issue', :mobile, testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1185' do issue = Resource::Issue.fabricate_via_browser_ui! Page::Project::Menu.perform(&:click_issues) @@ -19,13 +19,13 @@ module QA end end - it 'closes an issue', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1222' do + it 'closes an issue', :mobile, testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1222' do closed_issue.visit! Page::Project::Issue::Show.perform do |issue_page| issue_page.click_close_issue_button - expect(issue_page).to have_element(:reopen_issue_button) + expect(issue_page).to have_reopen_issue_button end Page::Project::Menu.perform(&:click_issues) diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/suggestions/batch_suggestion_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/suggestions/batch_suggestion_spec.rb index 1752513a831..96c9c9b55b4 100644 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/suggestions/batch_suggestion_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/suggestions/batch_suggestion_spec.rb @@ -2,7 +2,7 @@ module QA RSpec.describe 'Create' do - context 'Add batch suggestions to a Merge Request', :transient do + context 'Add batch suggestions to a Merge Request' do let(:project) do Resource::Project.fabricate_via_api! do |project| project.name = 'suggestions_project' @@ -46,13 +46,13 @@ module QA merge_request.visit! end - it 'applies multiple suggestions', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1838', quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/342131', type: :stale } do + it 'applies multiple suggestions', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1838' do Page::MergeRequest::Show.perform do |merge_request| merge_request.click_diffs_tab 4.times { merge_request.add_suggestion_to_batch } merge_request.apply_suggestion_with_message("Custom commit message") - expect(merge_request).to have_css('.badge-success', text: "Applied", count: 4) + expect(merge_request).to have_suggestions_applied(4) end end end diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/suggestions/custom_commit_suggestion_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/suggestions/custom_commit_suggestion_spec.rb index 339010cd1df..719006e87eb 100644 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/suggestions/custom_commit_suggestion_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/suggestions/custom_commit_suggestion_spec.rb @@ -48,7 +48,7 @@ module QA merge_request.click_diffs_tab merge_request.apply_suggestion_with_message(commit_message) - expect(merge_request).to have_css('.badge-success', text: 'Applied') + expect(merge_request).to have_suggestions_applied merge_request.click_commits_tab diff --git a/qa/qa/specs/features/browser_ui/3_create/web_ide/link_to_line_in_web_ide_spec.rb b/qa/qa/specs/features/browser_ui/3_create/web_ide/link_to_line_in_web_ide_spec.rb index c648fecf847..158e841514c 100644 --- a/qa/qa/specs/features/browser_ui/3_create/web_ide/link_to_line_in_web_ide_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/web_ide/link_to_line_in_web_ide_spec.rb @@ -21,7 +21,8 @@ module QA it 'can link to a specific line of code in Web IDE', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1844' do project.visit! - Page::Project::Show.perform(&:open_web_ide!) + # Open Web IDE by using a keyboard shortcut + Page::Project::Show.perform(&:open_web_ide_via_shortcut) Page::Project::WebIDE::Edit.perform do |ide| ide.select_file('app.js') diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb index 47117ae751f..e6910ad8592 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb @@ -1,10 +1,9 @@ # frozen_string_literal: true module QA - RSpec.describe 'Verify', :runner do + RSpec.describe 'Verify', :smoke, :runner do describe 'Pipeline creation and processing' do let(:executor) { "qa-runner-#{Time.now.to_i}" } - let(:max_wait) { 30 } let(:project) do Resource::Project.fabricate_via_api! do |project| @@ -21,11 +20,10 @@ module QA end after do - runner.remove_via_api! + [runner, project].each(&:remove_via_api!) end it 'users creates a pipeline which gets processed', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1601' do - # TODO: Convert back to :smoke once proved to be stable. Related issue: https://gitlab.com/gitlab-org/gitlab/-/issues/300909 Flow::Login.sign_in Resource::Repository::Commit.fabricate_via_api! do |commit| @@ -68,19 +66,21 @@ module QA Flow::Pipeline.visit_latest_pipeline - { - 'test-success': :passed, - 'test-failure': :failed, - 'test-tags-mismatch': :pending, - 'test-artifacts': :passed - }.each do |job, status| - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job(job) - end + aggregate_failures do + { + 'test-success': 'passed', + 'test-failure': 'failed', + 'test-tags-mismatch': 'pending', + 'test-artifacts': 'passed' + }.each do |job, status| + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job(job) + end - Page::Project::Job::Show.perform do |show| - expect(show).to public_send("be_#{status}") - show.click_element(:pipeline_path, Page::Project::Pipeline::Show) + Page::Project::Job::Show.perform do |show| + expect(show).to have_status(status), "Expected job status to be #{status} but got #{show.status_badge} instead." + show.click_element(:pipeline_path, Page::Project::Pipeline::Show) + end end end end diff --git a/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_omnibus_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_omnibus_spec.rb new file mode 100644 index 00000000000..51735d79fbd --- /dev/null +++ b/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_omnibus_spec.rb @@ -0,0 +1,191 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Package', :orchestrated, only: { pipeline: :main } do + describe 'Self-managed Container Registry' do + using RSpec::Parameterized::TableSyntax + + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'project-with-registry' + project.template_name = 'express' + project.visibility = :private + end + end + + let(:project_deploy_token) do + Resource::DeployToken.fabricate_via_browser_ui! do |deploy_token| + deploy_token.name = 'registry-deploy-token' + deploy_token.project = project + deploy_token.scopes = [ + :read_repository, + :read_package_registry, + :write_package_registry, + :read_registry, + :write_registry + ] + end + end + + let!(:runner) do + Resource::Runner.fabricate! do |runner| + runner.name = "qa-runner-#{Time.now.to_i}" + runner.tags = ["runner-for-#{project.name}"] + runner.executor = :docker + runner.project = project + end + end + + let(:personal_access_token) { Runtime::Env.personal_access_token } + + before do + Flow::Login.sign_in + project.visit! + end + + after do + runner.remove_via_api! + project.remove_via_api! + end + + where(:authentication_token_type, :token_name) do + :personal_access_token | 'Personal Access Token' + :project_deploy_token | 'Deploy Token' + :ci_job_token | 'Job Token' + end + + with_them do + let(:auth_token) do + case authentication_token_type + when :personal_access_token + "\"#{personal_access_token}\"" + when :project_deploy_token + "\"#{project_deploy_token.password}\"" + when :ci_job_token + '$CI_JOB_TOKEN' + end + end + + let(:auth_user) do + case authentication_token_type + when :personal_access_token + "$CI_REGISTRY_USER" + when :project_deploy_token + "\"#{project_deploy_token.username}\"" + when :ci_job_token + 'gitlab-ci-token' + end + end + + context "when tls is disabled" do + it "using a #{params[:token_name]}, pushes image and deletes tag", :registry do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files([{ + file_path: '.gitlab-ci.yml', + content: + <<~YAML + build: + image: docker:19.03.12 + stage: build + services: + - name: docker:19.03.12-dind + command: ["--insecure-registry=gitlab.test:5050"] + variables: + IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + script: + - docker login -u #{auth_user} -p #{auth_token} gitlab.test:5050 + - docker build -t $IMAGE_TAG . + - docker push $IMAGE_TAG + tags: + - "runner-for-#{project.name}" + YAML + }]) + end + + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('build') + end + + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + end + + Page::Project::Menu.perform(&:go_to_container_registry) + + Page::Project::Registry::Show.perform do |registry| + expect(registry).to have_registry_repository(project.path_with_namespace) + + registry.click_on_image(project.path_with_namespace) + expect(registry).to have_tag('master') + + registry.click_delete + expect(registry).not_to have_tag('master') + end + end + end + end + + context "when tls is enabled" do + it "pushes image and deletes tag", :registry_tls, testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/2378' do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files([{ + file_path: '.gitlab-ci.yml', + content: + <<~YAML + build: + image: docker:19.03.12 + stage: build + services: + - name: docker:19.03.12-dind + command: + - /bin/sh + - -c + - | + apk add --no-cache openssl + true | openssl s_client -showcerts -connect gitlab.test:5050 > /usr/local/share/ca-certificates/gitlab.test.crt + update-ca-certificates + dockerd-entrypoint.sh || exit + variables: + IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD gitlab.test:5050 + - docker build -t $IMAGE_TAG . + - docker push $IMAGE_TAG + tags: + - "runner-for-#{project.name}" + YAML + }]) + end + + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('build') + end + + Support::Retrier.retry_until(max_duration: 800, sleep_interval: 10) do + project.pipelines.last[:status] == 'success' + end + + Page::Project::Menu.perform(&:go_to_container_registry) + + Page::Project::Registry::Show.perform do |registry| + expect(registry).to have_registry_repository(project.path_with_namespace) + + registry.click_on_image(project.path_with_namespace) + expect(registry).to have_tag('master') + + registry.click_delete + expect(registry).not_to have_tag('master') + end + end + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/5_package/container_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_spec.rb index 65519cdebec..65519cdebec 100644 --- a/qa/qa/specs/features/browser_ui/5_package/container_registry_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_spec.rb diff --git a/qa/qa/specs/features/browser_ui/5_package/online_garbage_collection_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry/online_garbage_collection_spec.rb index 3ec76e8afad..82b7af8eba7 100644 --- a/qa/qa/specs/features/browser_ui/5_package/online_garbage_collection_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/container_registry/online_garbage_collection_spec.rb @@ -2,7 +2,8 @@ module QA RSpec.describe 'Package' do - describe 'Container Registry Online Garbage Collection', :registry_gc, only: { subdomain: %i[pre] } do + # TODO: Remove :requires_admin when the `Runtime::Feature.enable` method call is removed + describe 'Container Registry Online Garbage Collection', :registry_gc, :requires_admin, only: { subdomain: %i[pre] } do let(:group) { Resource::Group.fabricate_via_api! } let(:imported_project) do @@ -23,12 +24,12 @@ module QA STAGE_THREE_VALIDATION_DELAY: "6m" STAGE_FOUR_VALIDATION_DELAY: "12m" STAGE_FIVE_VALIDATION_DELAY: "12m" - + stages: - generate - build - test - + .base: &base image: docker:19 services: @@ -39,11 +40,11 @@ module QA DOCKER_TLS_VERIFY: 1 DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" before_script: - - until docker info; do sleep 1; done + - until docker info; do sleep 1; done - mkdir -p $GOPATH - mkdir -p $BUILD_CACHE - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - + test: stage: generate extends: .base @@ -64,6 +65,8 @@ module QA end before do + Runtime::Feature.enable(:paginatable_namespace_drop_down_for_project_creation) + Flow::Login.sign_in imported_project diff --git a/qa/qa/specs/features/browser_ui/5_package/container_registry_omnibus_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry_omnibus_spec.rb deleted file mode 100644 index 3d02c2884a2..00000000000 --- a/qa/qa/specs/features/browser_ui/5_package/container_registry_omnibus_spec.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: true - -module QA - RSpec.describe 'Package', :registry, :orchestrated, only: { pipeline: :main } do - describe 'Self-managed Container Registry' do - let(:project) do - Resource::Project.fabricate_via_api! do |project| - project.name = 'project-with-registry' - project.template_name = 'express' - end - end - - let!(:runner) do - Resource::Runner.fabricate! do |runner| - runner.name = "qa-runner-#{Time.now.to_i}" - runner.tags = ["runner-for-#{project.name}"] - runner.executor = :docker - runner.project = project - end - end - - before do - Flow::Login.sign_in - project.visit! - end - - after do - runner.remove_via_api! - end - - it "pushes image and deletes tag", testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1911' do - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.project = project - commit.commit_message = 'Add .gitlab-ci.yml' - commit.add_files([{ - file_path: '.gitlab-ci.yml', - content: - <<~YAML - build: - image: docker:19.03.12 - stage: build - services: - - name: docker:19.03.12-dind - command: - - /bin/sh - - -c - - | - apk add --no-cache openssl - true | openssl s_client -showcerts -connect gitlab.test:5050 > /usr/local/share/ca-certificates/gitlab.test.crt - update-ca-certificates - dockerd-entrypoint.sh || exit - variables: - IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD gitlab.test:5050 - - docker build -t $IMAGE_TAG . - - docker push $IMAGE_TAG - tags: - - "runner-for-#{project.name}" - YAML - }]) - end - - Flow::Pipeline.visit_latest_pipeline - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('build') - end - - Page::Project::Job::Show.perform do |job| - expect(job).to be_successful(timeout: 800) - end - - Page::Project::Menu.perform(&:go_to_container_registry) - - Page::Project::Registry::Show.perform do |registry| - expect(registry).to have_registry_repository(project.path_with_namespace) - - registry.click_on_image(project.path_with_namespace) - expect(registry).to have_tag('master') - - registry.click_delete - expect(registry).not_to have_tag('master') - end - end - end - end -end diff --git a/qa/qa/specs/features/browser_ui/5_package/dependency_proxy_spec.rb b/qa/qa/specs/features/browser_ui/5_package/dependency_proxy/dependency_proxy_spec.rb index ea7f7cc1c05..b941d5434df 100644 --- a/qa/qa/specs/features/browser_ui/5_package/dependency_proxy_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/dependency_proxy/dependency_proxy_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Package', :orchestrated, :registry do + RSpec.describe 'Package', :orchestrated, :registry, only: { pipeline: :main } do describe 'Dependency Proxy' do let(:project) do Resource::Project.fabricate_via_api! do |project| @@ -22,6 +22,7 @@ module QA let(:uri) { URI.parse(Runtime::Scenario.gitlab_address) } let(:gitlab_host_with_port) { "#{uri.host}:#{uri.port}" } let(:dependency_proxy_url) { "#{gitlab_host_with_port}/#{project.group.full_path}/dependency_proxy/containers" } + let(:image_sha) { 'alpine@sha256:c3d45491770c51da4ef58318e3714da686bc7165338b7ab5ac758e75c7455efb' } before do Flow::Login.sign_in @@ -56,22 +57,16 @@ module QA image: "#{docker_client_version}" services: - name: "#{docker_client_version}-dind" - command: - - /bin/sh - - -c - - | - apk add --no-cache openssl - true | openssl s_client -showcerts -connect gitlab.test:5050 > /usr/local/share/ca-certificates/gitlab.test.crt - update-ca-certificates - dockerd-entrypoint.sh || exit + command: ["--insecure-registry=gitlab.test:80"] before_script: - apk add curl jq grep - - docker login -u "$CI_DEPENDENCY_PROXY_USER" -p "$CI_DEPENDENCY_PROXY_PASSWORD" "$CI_DEPENDENCY_PROXY_SERVER" + - echo $CI_DEPENDENCY_PROXY_SERVER + - docker login -u "$CI_DEPENDENCY_PROXY_USER" -p "$CI_DEPENDENCY_PROXY_PASSWORD" gitlab.test:80 script: - - docker pull #{dependency_proxy_url}/alpine:latest + - docker pull #{dependency_proxy_url}/#{image_sha} - TOKEN=$(curl "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq --raw-output .token) - 'curl --head --header "Authorization: Bearer $TOKEN" "https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest" 2>&1' - - docker pull #{dependency_proxy_url}/alpine:latest + - docker pull #{dependency_proxy_url}/#{image_sha} - 'curl --head --header "Authorization: Bearer $TOKEN" "https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest" 2>&1' tags: - "runner-for-#{project.name}" @@ -95,7 +90,7 @@ module QA Page::Group::Menu.perform(&:go_to_dependency_proxy) Page::Group::DependencyProxy.perform do |index| - expect(index).to have_blob_count("Contains 2 blobs of images") + expect(index).to have_blob_count("Contains 1 blobs of images") end end end diff --git a/qa/qa/specs/features/browser_ui/5_package/maven_repository_spec.rb b/qa/qa/specs/features/browser_ui/5_package/maven_repository_spec.rb deleted file mode 100644 index bf1d2a04dba..00000000000 --- a/qa/qa/specs/features/browser_ui/5_package/maven_repository_spec.rb +++ /dev/null @@ -1,340 +0,0 @@ -# frozen_string_literal: true - -module QA - RSpec.describe 'Package', :orchestrated, :packages, :reliable, :object_storage do - describe 'Maven Repository' do - include Runtime::Fixtures - - let(:group_id) { 'com.gitlab.qa' } - let(:artifact_id) { "maven-#{SecureRandom.hex(8)}" } - let(:another_artifact_id) { "maven-#{SecureRandom.hex(8)}" } - let(:package_name) { "#{group_id}/#{artifact_id}".tr('.', '/') } - let(:auth_token) do - unless Page::Main::Menu.perform(&:signed_in?) - Flow::Login.sign_in - end - - Resource::PersonalAccessToken.fabricate!.token - end - - let(:project) do - Resource::Project.fabricate_via_api! do |project| - project.name = 'maven-package-project' - end - end - - let(:another_project) do - Resource::Project.fabricate_via_api! do |another_project| - another_project.name = 'another-maven-package-project' - another_project.group = project.group - end - end - - let(:package) do - Resource::Package.init do |package| - package.name = package_name - package.project = project - end - end - - let!(:runner) do - Resource::Runner.fabricate! do |runner| - runner.name = "qa-runner-#{Time.now.to_i}" - runner.tags = ["runner-for-#{project.group.name}"] - runner.executor = :docker - runner.token = project.group.runners_token - end - end - - let!(:gitlab_address_with_port) do - uri = URI.parse(Runtime::Scenario.gitlab_address) - "#{uri.scheme}://#{uri.host}:#{uri.port}" - end - - let(:pom_xml) do - { - file_path: 'pom.xml', - content: <<~XML - <project> - <groupId>#{group_id}</groupId> - <artifactId>#{artifact_id}</artifactId> - <version>1.0</version> - <modelVersion>4.0.0</modelVersion> - <repositories> - <repository> - <id>#{project.name}</id> - <url>#{gitlab_address_with_port}/api/v4/groups/#{project.group.id}/-/packages/maven</url> - </repository> - </repositories> - <distributionManagement> - <repository> - <id>#{project.name}</id> - <url>#{gitlab_address_with_port}/api/v4/projects/#{project.id}/packages/maven</url> - </repository> - <snapshotRepository> - <id>#{project.name}</id> - <url>#{gitlab_address_with_port}/api/v4/projects/#{project.id}/packages/maven</url> - </snapshotRepository> - </distributionManagement> - </project> - XML - } - end - - let(:pom_xml_another_project) do - { - file_path: 'pom.xml', - content: <<~XML - <project> - <groupId>#{group_id}</groupId> - <artifactId>#{another_artifact_id}</artifactId> - <version>1.0</version> - <modelVersion>4.0.0</modelVersion> - <repositories> - <repository> - <id>#{another_project.name}</id> - <url>#{gitlab_address_with_port}/api/v4/groups/#{another_project.group.id}/-/packages/maven</url> - </repository> - </repositories> - <distributionManagement> - <repository> - <id>#{another_project.name}</id> - <url>#{gitlab_address_with_port}/api/v4/projects/#{another_project.id}/packages/maven</url> - </repository> - <snapshotRepository> - <id>#{another_project.name}</id> - <url>#{gitlab_address_with_port}/api/v4/projects/#{another_project.id}/packages/maven</url> - </snapshotRepository> - </distributionManagement> - <dependencies> - <dependency> - <groupId>#{group_id}</groupId> - <artifactId>#{artifact_id}</artifactId> - <version>1.0</version> - </dependency> - </dependencies> - </project> - XML - } - end - - let(:settings_xml) do - { - file_path: 'settings.xml', - content: <<~XML - <settings xmlns="http://maven.apache.org/SETTINGS/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.1.0 http://maven.apache.org/xsd/settings-1.1.0.xsd"> - <servers> - <server> - <id>#{project.name}</id> - <configuration> - <httpHeaders> - <property> - <name>Private-Token</name> - <value>#{auth_token}</value> - </property> - </httpHeaders> - </configuration> - </server> - </servers> - </settings> - XML - } - end - - let(:gitlab_ci_deploy_yml) do - { - file_path: '.gitlab-ci.yml', - content: - <<~YAML - deploy: - image: maven:3.6-jdk-11 - script: - - 'mvn deploy -s settings.xml' - - "mvn dependency:get -Dartifact=#{group_id}:#{artifact_id}:1.0" - only: - - "#{project.default_branch}" - tags: - - "runner-for-#{project.group.name}" - YAML - } - end - - let(:gitlab_ci_install_yml) do - { - file_path: '.gitlab-ci.yml', - content: - <<~YAML - install: - image: maven:3.6-jdk-11 - script: - - "mvn install" - only: - - "#{project.default_branch}" - tags: - - "runner-for-#{another_project.group.name}" - YAML - } - end - - after do - runner.remove_via_api! - project.remove_via_api! - another_project.remove_via_api! - end - - it 'pushes and pulls a Maven package via CI and deletes it', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1627' do - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.project = project - commit.commit_message = 'Add .gitlab-ci.yml' - commit.add_files([ - gitlab_ci_deploy_yml, - settings_xml, - pom_xml - ]) - end - - project.visit! - Flow::Pipeline.visit_latest_pipeline - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('deploy') - end - - Page::Project::Job::Show.perform do |job| - expect(job).to be_successful(timeout: 800) - end - - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.project = another_project - commit.commit_message = 'Add .gitlab-ci.yml' - commit.add_files([ - gitlab_ci_install_yml, - pom_xml_another_project - ]) - end - - another_project.visit! - Flow::Pipeline.visit_latest_pipeline - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('install') - end - - Page::Project::Job::Show.perform do |job| - expect(job).to be_successful(timeout: 800) - end - - project.visit! - - Page::Project::Menu.perform(&:click_packages_link) - - Page::Project::Packages::Index.perform do |index| - expect(index).to have_package(package_name) - - index.click_package(package_name) - end - - Page::Project::Packages::Show.perform do |show| - expect(show).to have_package_info(package_name, "1.0") - show.click_delete - end - - Page::Project::Packages::Index.perform do |index| - expect(index).to have_content("Package deleted successfully") - expect(index).not_to have_package(package_name) - end - end - - context 'when "allow duplicate" setting is disabled' do - before do - Flow::Login.sign_in - - project.group.visit! - - Page::Group::Menu.perform(&:go_to_package_settings) - Page::Group::Settings::PackageRegistries.perform(&:set_allow_duplicates_disabled) - end - - it 'prevents users from publishing duplicate Maven packages at the group level', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1830' do - with_fixtures([pom_xml, settings_xml]) do |dir| - Service::DockerRun::Maven.new(dir).publish! - end - - project.visit! - Page::Project::Menu.perform(&:click_packages_link) - - Page::Project::Packages::Index.perform do |index| - expect(index).to have_package(package_name) - end - - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.project = another_project - commit.commit_message = 'Add .gitlab-ci.yml' - commit.add_files([ - gitlab_ci_deploy_yml, - settings_xml, - pom_xml - ]) - end - - another_project.visit! - Flow::Pipeline.visit_latest_pipeline - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('deploy') - end - - Page::Project::Job::Show.perform do |job| - expect(job).not_to be_successful(timeout: 800) - end - end - end - - context 'when "allow duplicate" setting is enabled' do - before do - Flow::Login.sign_in - - project.group.visit! - - Page::Group::Menu.perform(&:go_to_package_settings) - Page::Group::Settings::PackageRegistries.perform(&:set_allow_duplicates_enabled) - end - - it 'allows users to publish duplicate Maven packages at the group level', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1829' do - with_fixtures([pom_xml, settings_xml]) do |dir| - Service::DockerRun::Maven.new(dir).publish! - end - - project.visit! - Page::Project::Menu.perform(&:click_packages_link) - - Page::Project::Packages::Index.perform do |index| - expect(index).to have_package(package_name) - end - - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.project = another_project - commit.commit_message = 'Add .gitlab-ci.yml' - commit.add_files([ - gitlab_ci_deploy_yml, - settings_xml, - pom_xml - ]) - end - - another_project.visit! - Flow::Pipeline.visit_latest_pipeline - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('deploy') - end - - Page::Project::Job::Show.perform do |job| - expect(job).to be_successful(timeout: 800) - end - end - end - end - end -end diff --git a/qa/qa/specs/features/browser_ui/5_package/composer_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/composer_registry_spec.rb index 9ddf485870d..9ddf485870d 100644 --- a/qa/qa/specs/features/browser_ui/5_package/composer_registry_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/composer_registry_spec.rb diff --git a/qa/qa/specs/features/browser_ui/5_package/conan_repository_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/conan_repository_spec.rb index a8f1fc2a7de..126be22d760 100644 --- a/qa/qa/specs/features/browser_ui/5_package/conan_repository_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/conan_repository_spec.rb @@ -17,7 +17,7 @@ module QA let(:package) do Resource::Package.init do |package| - package.name = 'conantest' + package.name = "conantest-#{SecureRandom.hex(8)}" package.project = project end end diff --git a/qa/qa/specs/features/browser_ui/5_package/generic_repository_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/generic_repository_spec.rb index 2e5fa2c2904..86aca120eed 100644 --- a/qa/qa/specs/features/browser_ui/5_package/generic_repository_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/generic_repository_spec.rb @@ -11,7 +11,7 @@ module QA let(:package) do Resource::Package.init do |package| - package.name = "my_package" + package.name = "my_package-#{SecureRandom.hex(8)}" package.project = project end end @@ -36,13 +36,13 @@ module QA upload: stage: upload script: - - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file file.txt ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/my_package/0.0.1/file.txt' + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file file.txt ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/#{package.name}/0.0.1/file.txt' tags: - "runner-for-#{project.name}" download: stage: download script: - - 'wget --header="JOB-TOKEN: $CI_JOB_TOKEN" ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/my_package/0.0.1/file.txt -O file_downloaded.txt' + - 'wget --header="JOB-TOKEN: $CI_JOB_TOKEN" ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/#{package.name}/0.0.1/file.txt -O file_downloaded.txt' tags: - "runner-for-#{project.name}" YAML diff --git a/qa/qa/specs/features/browser_ui/5_package/helm_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/helm_registry_spec.rb index fe52fd03ad8..3f5e8b1a630 100644 --- a/qa/qa/specs/features/browser_ui/5_package/helm_registry_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/helm_registry_spec.rb @@ -6,7 +6,7 @@ module QA include Runtime::Fixtures include_context 'packages registry qa scenario' - let(:package_name) { 'gitlab_qa_helm' } + let(:package_name) { "gitlab_qa_helm-#{SecureRandom.hex(8)}" } let(:package_version) { '1.3.7' } let(:package_type) { 'helm' } diff --git a/qa/qa/specs/features/browser_ui/5_package/maven_gradle_repository_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/maven_gradle_repository_spec.rb index ec9feca84b9..2aa93de0b9e 100644 --- a/qa/qa/specs/features/browser_ui/5_package/maven_gradle_repository_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/maven_gradle_repository_spec.rb @@ -8,7 +8,7 @@ module QA include_context 'packages registry qa scenario' let(:group_id) { 'com.gitlab.qa' } - let(:artifact_id) { 'maven_gradle' } + let(:artifact_id) { "maven_gradle-#{SecureRandom.hex(8)}" } let(:package_name) { "#{group_id}/#{artifact_id}".tr('.', '/') } let(:package_version) { '1.3.7' } let(:package_type) { 'maven_gradle' } diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/maven_repository_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/maven_repository_spec.rb new file mode 100644 index 00000000000..f42093bffcd --- /dev/null +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/maven_repository_spec.rb @@ -0,0 +1,317 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Package', :orchestrated, :packages, :object_storage do + describe 'Maven Repository' do + using RSpec::Parameterized::TableSyntax + include Runtime::Fixtures + include_context 'packages registry qa scenario' + + let(:group_id) { 'com.gitlab.qa' } + let(:artifact_id) { "maven-#{SecureRandom.hex(8)}" } + let(:package_name) { "#{group_id}/#{artifact_id}".tr('.', '/') } + let(:package_version) { '1.3.7' } + let(:package_type) { 'maven' } + + let(:package_gitlab_ci_file) do + { + file_path: '.gitlab-ci.yml', + content: + <<~YAML + deploy: + image: maven:3.6-jdk-11 + script: + - 'mvn deploy -s settings.xml' + only: + - "#{package_project.default_branch}" + tags: + - "runner-for-#{package_project.group.name}" + YAML + } + end + + let(:package_pom_file) do + { + file_path: 'pom.xml', + content: <<~XML + <project> + <groupId>#{group_id}</groupId> + <artifactId>#{artifact_id}</artifactId> + <version>#{package_version}</version> + <modelVersion>4.0.0</modelVersion> + <repositories> + <repository> + <id>#{package_project.name}</id> + <url>#{gitlab_address_with_port}/api/v4/groups/#{package_project.group.id}/-/packages/maven</url> + </repository> + </repositories> + <distributionManagement> + <repository> + <id>#{package_project.name}</id> + <url>#{gitlab_address_with_port}/api/v4/projects/#{package_project.id}/packages/maven</url> + </repository> + <snapshotRepository> + <id>#{package_project.name}</id> + <url>#{gitlab_address_with_port}/api/v4/projects/#{package_project.id}/packages/maven</url> + </snapshotRepository> + </distributionManagement> + </project> + XML + } + end + + let(:client_gitlab_ci_file) do + { + file_path: '.gitlab-ci.yml', + content: + <<~YAML + install: + image: maven:3.6-jdk-11 + script: + - "mvn install -s settings.xml" + only: + - "#{client_project.default_branch}" + tags: + - "runner-for-#{client_project.group.name}" + YAML + } + end + + let(:client_pom_file) do + { + file_path: 'pom.xml', + content: <<~XML + <project> + <groupId>#{group_id}</groupId> + <artifactId>maven_client</artifactId> + <version>1.0</version> + <modelVersion>4.0.0</modelVersion> + <repositories> + <repository> + <id>#{package_project.name}</id> + <url>#{gitlab_address_with_port}/api/v4/groups/#{package_project.group.id}/-/packages/maven</url> + </repository> + </repositories> + <dependencies> + <dependency> + <groupId>#{group_id}</groupId> + <artifactId>#{artifact_id}</artifactId> + <version>#{package_version}</version> + </dependency> + </dependencies> + </project> + XML + } + end + + let(:settings_xml_with_pat) do + { + file_path: 'settings.xml', + content: <<~XML + <settings xmlns="http://maven.apache.org/SETTINGS/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.1.0 http://maven.apache.org/xsd/settings-1.1.0.xsd"> + <servers> + <server> + <id>#{package_project.name}</id> + <configuration> + <httpHeaders> + <property> + <name>Private-Token</name> + <value>#{personal_access_token}</value> + </property> + </httpHeaders> + </configuration> + </server> + </servers> + </settings> + XML + } + end + + where(:authentication_token_type, :maven_header_name) do + :personal_access_token | 'Private-Token' + :ci_job_token | 'Job-Token' + :project_deploy_token | 'Deploy-Token' + end + + with_them do + let(:token) do + case authentication_token_type + when :personal_access_token + personal_access_token + when :ci_job_token + '${env.CI_JOB_TOKEN}' + when :project_deploy_token + project_deploy_token.password + end + end + + let(:settings_xml) do + { + file_path: 'settings.xml', + content: <<~XML + <settings xmlns="http://maven.apache.org/SETTINGS/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.1.0 http://maven.apache.org/xsd/settings-1.1.0.xsd"> + <servers> + <server> + <id>#{package_project.name}</id> + <configuration> + <httpHeaders> + <property> + <name>#{maven_header_name}</name> + <value>#{token}</value> + </property> + </httpHeaders> + </configuration> + </server> + </servers> + </settings> + XML + } + end + + it "pushes and pulls a maven package via maven using #{params[:authentication_token_type]}" do + # pushing + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = package_project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files([ + package_gitlab_ci_file, + package_pom_file, + settings_xml + ]) + end + + package_project.visit! + + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('deploy') + end + + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + end + + Page::Project::Menu.perform(&:click_packages_link) + + Page::Project::Packages::Index.perform do |index| + expect(index).to have_package(package_name) + + index.click_package(package_name) + end + + Page::Project::Packages::Show.perform do |show| + expect(show).to have_package_info(package_name, package_version) + end + + # pulling + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = client_project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files([ + client_gitlab_ci_file, + client_pom_file, + settings_xml + ]) + end + + client_project.visit! + + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('install') + end + + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + end + end + + context 'duplication setting' do + before do + package_project.group.visit! + + Page::Group::Menu.perform(&:go_to_package_settings) + end + + context 'when disabled' do + before do + Page::Group::Settings::PackageRegistries.perform(&:set_allow_duplicates_disabled) + end + + it "prevents users from publishing group level Maven packages duplicates using #{params[:authentication_token_type]}" do + create_duplicated_package + + push_duplicated_package + + client_project.visit! + + show_latest_deploy_job + + Page::Project::Job::Show.perform do |job| + expect(job).not_to be_successful(timeout: 800) + end + end + end + + context 'when enabled' do + before do + Page::Group::Settings::PackageRegistries.perform(&:set_allow_duplicates_enabled) + end + + it "allows users to publish group level Maven packages duplicates using #{params[:authentication_token_type]}" do + create_duplicated_package + + push_duplicated_package + + show_latest_deploy_job + + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + end + end + end + + def create_duplicated_package + with_fixtures([package_pom_file, settings_xml_with_pat]) do |dir| + Service::DockerRun::Maven.new(dir).publish! + end + + package_project.visit! + + Page::Project::Menu.perform(&:click_packages_link) + + Page::Project::Packages::Index.perform do |index| + expect(index).to have_package(package_name) + end + end + + def push_duplicated_package + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = client_project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files([ + package_gitlab_ci_file, + package_pom_file, + settings_xml + ]) + end + end + + def show_latest_deploy_job + client_project.visit! + + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('deploy') + end + end + end + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/5_package/npm_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb index 5a3b4388f0c..f2b1c1b01a0 100644 --- a/qa/qa/specs/features/browser_ui/5_package/npm_registry_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true module QA - RSpec.describe 'Package', :orchestrated, :packages, :reliable, :object_storage do - describe 'npm registry' do + RSpec.describe 'Package Registry', :orchestrated, :packages, :reliable, :object_storage do + describe 'npm instance level endpoint' do using RSpec::Parameterized::TableSyntax include Runtime::Fixtures @@ -19,6 +19,11 @@ module QA Resource::DeployToken.fabricate_via_browser_ui! do |deploy_token| deploy_token.name = 'npm-deploy-token' deploy_token.project = project + deploy_token.scopes = [ + :read_repository, + :read_package_registry, + :write_package_registry + ] end end @@ -28,13 +33,13 @@ module QA let!(:project) do Resource::Project.fabricate_via_api! do |project| - project.name = 'npm-project' + project.name = 'npm-instace-level-publish' end end let!(:another_project) do Resource::Project.fabricate_via_api! do |another_project| - another_project.name = 'npm-another-project' + another_project.name = 'npm-instance-level-install' another_project.template_name = 'express' another_project.group = project.group end @@ -54,7 +59,7 @@ module QA file_path: '.gitlab-ci.yml', content: <<~YAML - image: node:14-buster + image: node:latest stages: - deploy @@ -62,6 +67,7 @@ module QA deploy: stage: deploy script: + - echo "//${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=#{auth_token}">.npmrc - npm publish only: - "#{project.default_branch}" @@ -119,7 +125,7 @@ module QA let(:package) do Resource::Package.init do |package| - package.name = "@#{registry_scope}/#{project.name}" + package.name = "@#{registry_scope}/#{project.name}-#{SecureRandom.hex(8)}" package.project = project end end @@ -149,23 +155,12 @@ module QA end end - let(:npmrc) do - { - file_path: '.npmrc', - content: <<~NPMRC - //#{gitlab_host_with_port}/api/v4/projects/#{project.id}/packages/npm/:_authToken=#{auth_token} - @#{registry_scope}:registry=#{gitlab_address_with_port}/api/v4/projects/#{project.id}/packages/npm/ - NPMRC - } - end - - it "push and pull a npm package via CI using a #{params[:token_name]}", testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1772' do + it "push and pull a npm package via CI using a #{params[:token_name]}" do Resource::Repository::Commit.fabricate_via_api! do |commit| commit.project = project commit.commit_message = 'Add .gitlab-ci.yml' commit.add_files([ gitlab_ci_deploy_yaml, - npmrc, package_json ]) end diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb new file mode 100644 index 00000000000..832f8c7f72c --- /dev/null +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_project_level_spec.rb @@ -0,0 +1,197 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Package Registry', :orchestrated, :packages, :reliable, :object_storage do + describe 'npm project level endpoint' do + using RSpec::Parameterized::TableSyntax + include Runtime::Fixtures + + let!(:registry_scope) { Runtime::Namespace.sandbox_name } + let!(:personal_access_token) do + unless Page::Main::Menu.perform(&:signed_in?) + Flow::Login.sign_in + end + + Resource::PersonalAccessToken.fabricate!.token + end + + let(:project_deploy_token) do + Resource::DeployToken.fabricate_via_browser_ui! do |deploy_token| + deploy_token.name = 'npm-deploy-token' + deploy_token.project = project + deploy_token.scopes = [ + :read_repository, + :read_package_registry, + :write_package_registry + ] + end + end + + let(:uri) { URI.parse(Runtime::Scenario.gitlab_address) } + let(:gitlab_address_with_port) { "#{uri.scheme}://#{uri.host}:#{uri.port}" } + let(:gitlab_host_with_port) { "#{uri.host}:#{uri.port}" } + + let!(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'npm-project-level' + end + end + + let!(:runner) do + Resource::Runner.fabricate! do |runner| + runner.name = "qa-runner-#{Time.now.to_i}" + runner.tags = ["runner-for-#{project.name}"] + runner.executor = :docker + runner.project = project + end + end + + let(:gitlab_ci_yaml) do + { + file_path: '.gitlab-ci.yml', + content: + <<~YAML + image: node:latest + + stages: + - deploy + - install + + deploy: + stage: deploy + script: + - echo "//${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=#{auth_token}">.npmrc + - npm publish + only: + - "#{project.default_branch}" + tags: + - "runner-for-#{project.name}" + install: + stage: install + script: + - "npm config set @#{registry_scope}:registry #{gitlab_address_with_port}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/" + - "npm install #{package.name}" + cache: + key: ${CI_BUILD_REF_NAME} + paths: + - node_modules/ + artifacts: + paths: + - node_modules/ + only: + - "#{project.default_branch}" + tags: + - "runner-for-#{project.name}" + YAML + } + end + + let(:package_json) do + { + file_path: 'package.json', + content: <<~JSON + { + "name": "@#{registry_scope}/mypackage", + "version": "1.0.0", + "description": "Example package for GitLab npm registry", + "publishConfig": { + "@#{registry_scope}:registry": "#{gitlab_address_with_port}/api/v4/projects/#{project.id}/packages/npm/" + } + } + JSON + } + end + + let(:package) do + Resource::Package.init do |package| + package.name = "@#{registry_scope}/mypackage-#{SecureRandom.hex(8)}" + package.project = project + end + end + + after do + package.remove_via_api! + runner.remove_via_api! + project.remove_via_api! + end + + where(:authentication_token_type, :token_name) do + :personal_access_token | 'Personal Access Token' + :ci_job_token | 'CI Job Token' + :project_deploy_token | 'Deploy Token' + end + + with_them do + let(:auth_token) do + case authentication_token_type + when :personal_access_token + "\"#{personal_access_token}\"" + when :ci_job_token + '${CI_JOB_TOKEN}' + when :project_deploy_token + "\"#{project_deploy_token.password}\"" + end + end + + it "push and pull a npm package via CI using a #{params[:token_name]}", quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/344537', type: :investigating } do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files([ + gitlab_ci_yaml, + package_json + ]) + end + + project.visit! + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('deploy') + end + + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + end + + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('install') + end + + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + job.click_browse_button + end + + Page::Project::Artifact::Show.perform do |artifacts| + artifacts.go_to_directory('node_modules') + artifacts.go_to_directory("@#{registry_scope}") + expect(artifacts).to have_content("mypackage") + end + + project.visit! + Page::Project::Menu.perform(&:click_packages_link) + + Page::Project::Packages::Index.perform do |index| + expect(index).to have_package(package.name) + + index.click_package(package.name) + end + + Page::Project::Packages::Show.perform do |show| + expect(show).to have_package_info(package.name, "1.0.0") + + show.click_delete + end + + Page::Project::Packages::Index.perform do |index| + expect(index).to have_content("Package deleted successfully") + expect(index).not_to have_package(package.name) + end + end + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/5_package/nuget_repository_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/nuget_repository_spec.rb index 8a6752ed817..0b4825715c1 100644 --- a/qa/qa/specs/features/browser_ui/5_package/nuget_repository_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/nuget_repository_spec.rb @@ -85,7 +85,7 @@ module QA end end - it "publishes a nuget package at the project level, installs and deletes it at the group level using a #{params[:token_name]}", testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1073' do + it "publishes a nuget package at the project level, installs and deletes it at the group level using a #{params[:token_name]}" do Flow::Login.sign_in Resource::Repository::Commit.fabricate_via_api! do |commit| diff --git a/qa/qa/specs/features/browser_ui/5_package/pypi_repository_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/pypi_repository_spec.rb index dfc9202ebed..e727a89a584 100644 --- a/qa/qa/specs/features/browser_ui/5_package/pypi_repository_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/pypi_repository_spec.rb @@ -12,7 +12,7 @@ module QA let(:package) do Resource::Package.init do |package| - package.name = 'mypypipackage' + package.name = "mypypipackage-#{SecureRandom.hex(8)}" package.project = project end end @@ -57,7 +57,7 @@ module QA install: stage: install script: - - "pip install mypypipackage --no-deps --index-url #{uri.scheme}://#{personal_access_token}:#{personal_access_token}@#{gitlab_host_with_port}/api/v4/projects/${CI_PROJECT_ID}/packages/pypi/simple --trusted-host #{gitlab_host_with_port}" + - "pip install #{package.name} --no-deps --index-url #{uri.scheme}://#{personal_access_token}:#{personal_access_token}@#{gitlab_host_with_port}/api/v4/projects/${CI_PROJECT_ID}/packages/pypi/simple --trusted-host #{gitlab_host_with_port}" tags: - "runner-for-#{project.name}" @@ -70,7 +70,7 @@ module QA import setuptools setuptools.setup( - name="mypypipackage", + name="#{package.name}", version="0.0.1", author="Example Author", author_email="author@example.com", diff --git a/qa/qa/specs/features/browser_ui/5_package/rubygems_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/rubygems_registry_spec.rb index 9a45b072eed..ecf14a25b8d 100644 --- a/qa/qa/specs/features/browser_ui/5_package/rubygems_registry_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/rubygems_registry_spec.rb @@ -13,7 +13,7 @@ module QA let(:package) do Resource::Package.init do |package| - package.name = 'mygem' + package.name = "mygem-#{SecureRandom.hex(8)}" package.project = project end end @@ -46,17 +46,9 @@ module QA it 'publishes and deletes a Ruby gem', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1906' do Flow::Login.sign_in - Resource::Repository::ProjectPush.fabricate! do |push| - push.project = project - push.directory = Pathname - .new(__dir__) - .join('../../../../fixtures/rubygems_package') - push.commit_message = 'RubyGems package' - end - Resource::Repository::Commit.fabricate_via_api! do |commit| commit.project = project - commit.commit_message = 'Add mygem.gemspec' + commit.commit_message = 'Add package files' commit.add_files( [ { @@ -74,8 +66,8 @@ module QA echo "#{gitlab_address_with_port}/api/v4/projects/${CI_PROJECT_ID}/packages/rubygems: '${CI_JOB_TOKEN}'" >> ~/.gem/credentials - chmod 0600 ~/.gem/credentials script: - - gem build mygem - - gem push mygem-0.0.1.gem --host #{gitlab_address_with_port}/api/v4/projects/${CI_PROJECT_ID}/packages/rubygems + - gem build #{package.name} + - gem push #{package.name}-0.0.1.gem --host #{gitlab_address_with_port}/api/v4/projects/${CI_PROJECT_ID}/packages/rubygems tags: - "runner-for-#{project.name}" YAML @@ -90,6 +82,52 @@ module QA end end RUBY + }, + { + file_path: "#{package.name}.gemspec", + content: + <<~RUBY + # frozen_string_literal: true + + Gem::Specification.new do |s| + s.name = '#{package.name}' + s.authors = ['Tanuki Steve', 'Hal 9000'] + s.author = 'Tanuki Steve' + s.version = '0.0.1' + s.date = '2011-09-29' + s.summary = 'this is a test package' + s.files = ['lib/hello_gem.rb'] + s.require_paths = ['lib'] + + s.description = 'A test package for GitLab.' + s.email = 'tanuki@not_real.com' + s.homepage = 'https://gitlab.com/ruby-co/my-package' + s.license = 'MIT' + + s.metadata = { + 'bug_tracker_uri' => 'https://gitlab.com/ruby-co/my-package/issues', + 'changelog_uri' => 'https://gitlab.com/ruby-co/my-package/CHANGELOG.md', + 'documentation_uri' => 'https://gitlab.com/ruby-co/my-package/docs', + 'mailing_list_uri' => 'https://gitlab.com/ruby-co/my-package/mailme', + 'source_code_uri' => 'https://gitlab.com/ruby-co/my-package' + } + + s.bindir = 'bin' + s.platform = Gem::Platform::RUBY + s.post_install_message = 'Installed, thank you!' + s.rdoc_options = ['--main'] + s.required_ruby_version = '>= 2.7.0' + s.required_rubygems_version = '>= 1.8.11' + s.requirements = 'A high powered server or calculator' + s.rubygems_version = '1.8.09' + + s.add_dependency 'dependency_1', '~> 1.2.3' + s.add_dependency 'dependency_2', '3.0.0' + s.add_dependency 'dependency_3', '>= 1.0.0' + s.add_dependency 'dependency_4' + end + + RUBY } ] ) diff --git a/qa/qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb b/qa/qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb index 23625ab645d..81ccc585cf9 100644 --- a/qa/qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb +++ b/qa/qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb @@ -12,6 +12,7 @@ module QA deploy_token = Resource::DeployToken.fabricate_via_browser_ui! do |resource| resource.name = deploy_token_name resource.expires_at = one_week_from_now + resource.scopes = [:read_repository] end expect(deploy_token.username.length).to be > 0 diff --git a/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb b/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb index 655c806a37a..e20b76f6bf8 100644 --- a/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb +++ b/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true module QA - RSpec.describe 'Configure' do + RSpec.describe 'Configure', only: { subdomain: :staging } do let(:project) do Resource::Project.fabricate_via_api! do |project| - project.name = Runtime::Env.auto_devops_project_name || 'autodevops-project' + project.name = 'autodevops-project' project.auto_devops_enabled = true end end @@ -13,35 +13,24 @@ module QA disable_optional_jobs(project) end - describe 'Auto DevOps support', :orchestrated, :kubernetes, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/251090', type: :stale } do + describe 'Auto DevOps support' do context 'when rbac is enabled' do let(:cluster) { Service::KubernetesCluster.new.create! } after do cluster&.remove! + project.remove_via_api! end it 'runs auto devops', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1422' do Flow::Login.sign_in - # Set an application secret CI variable (prefixed with K8S_SECRET_) - Resource::CiVariable.fabricate! do |resource| - resource.project = project - resource.key = 'K8S_SECRET_OPTIONAL_MESSAGE' - resource.value = 'you_can_see_this_variable' - resource.masked = false - end - - # Connect K8s cluster Resource::KubernetesCluster::ProjectCluster.fabricate! do |k8s_cluster| k8s_cluster.project = project k8s_cluster.cluster = cluster k8s_cluster.install_ingress = true - k8s_cluster.install_prometheus = true - k8s_cluster.install_runner = true end - # Create Auto DevOps compatible repo Resource::Repository::ProjectPush.fabricate! do |push| push.project = project push.directory = Pathname @@ -78,46 +67,6 @@ module QA job.click_element(:pipeline_path) end - - Page::Project::Menu.perform(&:go_to_deployments_environments) - Page::Project::Deployments::Environments::Index.perform do |index| - index.click_environment_link('production') - end - Page::Project::Deployments::Environments::Show.perform do |show| - show.view_deployment do - expect(page).to have_content('Hello World!') - expect(page).to have_content('you_can_see_this_variable') - end - end - end - end - end - - describe 'Auto DevOps', :smoke do - before do - Flow::Login.sign_in - - project.visit! - - Page::Project::Menu.perform(&:go_to_ci_cd_settings) - Page::Project::Settings::CiCd.perform(&:expand_auto_devops) - Page::Project::Settings::AutoDevops.perform(&:enable_autodevops) - - # Create AutoDevOps repo - Resource::Repository::ProjectPush.fabricate! do |push| - push.project = project - push.directory = Pathname - .new(__dir__) - .join('../../../../../fixtures/auto_devops_rack') - push.commit_message = 'Create AutoDevOps compatible Project' - end - end - - it 'runs an AutoDevOps pipeline', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1564' do - Flow::Pipeline.visit_latest_pipeline - - Page::Project::Pipeline::Show.perform do |pipeline| - expect(pipeline).to have_tag('Auto DevOps') end end end @@ -128,7 +77,8 @@ module QA %w[ CODE_QUALITY_DISABLED LICENSE_MANAGEMENT_DISABLED SAST_DISABLED DAST_DISABLED DEPENDENCY_SCANNING_DISABLED - CONTAINER_SCANNING_DISABLED + CONTAINER_SCANNING_DISABLED BROWSER_PERFORMANCE_DISABLED + SECRET_DETECTION_DISABLED ].each do |key| Resource::CiVariable.fabricate_via_api! do |resource| resource.project = project diff --git a/qa/qa/specs/helpers/context_selector.rb b/qa/qa/specs/helpers/context_selector.rb index 57665babf68..9ac79ad6196 100644 --- a/qa/qa/specs/helpers/context_selector.rb +++ b/qa/qa/specs/helpers/context_selector.rb @@ -45,11 +45,11 @@ module QA opts[:subdomain] = case option[:subdomain] when Array - "(#{option[:subdomain].join("|")})." + "(#{option[:subdomain].join("|")})\\." when Regexp option[:subdomain] else - "(#{option[:subdomain]})." + "(#{option[:subdomain]})\\." end end end diff --git a/qa/qa/support/fabrication_tracker.rb b/qa/qa/support/fabrication_tracker.rb new file mode 100644 index 00000000000..3238cc5b0db --- /dev/null +++ b/qa/qa/support/fabrication_tracker.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module QA + module Support + # Threadsafe fabrication time tracker + # + # Ongoing fabrication is added to callstack by start_fabrication and taken out by finish_fabrication + # + # Fabrication runtime is saved only for the first fabrication in the stack to properly represent the real time + # fabrications might take as top level fabrication runtime will always include nested fabrications runtime + # + class FabricationTracker + class << self + # Start fabrication and increment ongoing fabrication count + # + # @return [void] + def start_fabrication + Thread.current[:fabrications_ongoing] = 0 unless Thread.current.key?(:fabrications_ongoing) + + Thread.current[:fabrications_ongoing] += 1 + end + + # Finish fabrication and decrement ongoing fabrication count + # + # @return [void] + def finish_fabrication + Thread.current[:fabrications_ongoing] -= 1 + end + + # Save fabrication time if it's first in fabrication stack + # + # @param [Symbol] type + # @param [Symbol] time + # @return [void] + def save_fabrication(type, time) + return unless Thread.current.key?(type) + return unless top_level_fabrication? + + Thread.current[type] += time + end + + private + + # Check if current fabrication is the only one in the stack + # + # @return [Boolean] + def top_level_fabrication? + Thread.current[:fabrications_ongoing] == 1 + end + end + end + end +end diff --git a/qa/qa/support/formatters/test_stats_formatter.rb b/qa/qa/support/formatters/test_stats_formatter.rb index 0484bd7f90f..b54b6a51d11 100644 --- a/qa/qa/support/formatters/test_stats_formatter.rb +++ b/qa/qa/support/formatters/test_stats_formatter.rb @@ -57,6 +57,8 @@ module QA # @return [Hash] def test_stats(example) file_path = example.metadata[:file_path].gsub('./qa/specs/features', '') + api_fabrication = ((example.metadata[:api_fabrication] || 0) * 1000).round + ui_fabrication = ((example.metadata[:browser_ui_fabrication] || 0) * 1000).round { name: 'test-stats', @@ -76,6 +78,9 @@ module QA fields: { id: example.id, run_time: (example.execution_result.run_time * 1000).round, + api_fabrication: api_fabrication, + ui_fabrication: ui_fabrication, + total_fabrication: api_fabrication + ui_fabrication, retry_attempts: example.metadata[:retry_attempts] || 0, job_url: QA::Runtime::Env.ci_job_url, pipeline_url: env('CI_PIPELINE_URL'), @@ -98,14 +103,18 @@ module QA # # @return [String] def job_name - @job_name ||= QA::Runtime::Env.ci_job_name.gsub(%r{ \d{1,2}/\d{1,2}}, '') + @job_name ||= QA::Runtime::Env.ci_job_name&.gsub(%r{ \d{1,2}/\d{1,2}}, '') end # Single common timestamp for all exported example metrics to keep data points consistently grouped # # @return [Time] def time - @time ||= DateTime.strptime(env('CI_PIPELINE_CREATED_AT')).to_time + @time ||= begin + return Time.now unless env('CI_PIPELINE_CREATED_AT') + + DateTime.strptime(env('CI_PIPELINE_CREATED_AT')).to_time + end end # Is a merge request execution diff --git a/qa/qa/support/helpers/plan.rb b/qa/qa/support/helpers/plan.rb new file mode 100644 index 00000000000..298a6d3f036 --- /dev/null +++ b/qa/qa/support/helpers/plan.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module QA + module Support + module Helpers + module Plan + FREE = { name: 'free', price: 0, yearly_price: 0, ci_minutes: 400 }.freeze + + PREMIUM = { + plan_id: '2c92a00d76f0d5060176f2fb0a5029ff', + rate_charge_id: '2c92a00d76f0d5060176f2fb0a672a02', + name: 'premium', + price: 19, + yearly_price: 228, + ci_minutes: 10000 + }.freeze + + PREMIUM_SELF_MANAGED = { + plan_id: '2c92a01176f0d50a0176f3043c4d4a53', + rate_charge_id: '2c92a01176f0d50a0176f3043c6a4a58', + name: 'premium', + price: 19, + yearly_price: 228 + }.freeze + + ULTIMATE = { + plan_id: '2c92a0ff76f0d5250176f2f8c86f305a', + rate_charge_id: '2c92a0ff76f0d5250176f2f8c896305c', + name: 'ultimate', + price: 99, + yearly_price: 1188, + ci_minutes: 50000 + }.freeze + + ULTIMATE_SELF_MANAGED = { + plan_id: '2c92a00c76f0c6c20176f2f9328b33c9', + rate_charge_id: '2c92a00c76f0c6c20176f2fcbb645b5f', + name: 'ultimate', + price: 99, + yearly_price: 1188 + }.freeze + + CI_MINUTES = { + plan_id: '2c92a0086a07f4a8016a2c0a1f7b4b4c', + rate_charge_id: '2c92a0fd6a07f4c6016a2c0af07c3f21', + name: 'ci_minutes', + price: 10, + ci_minutes: 1000 + }.freeze + + STORAGE = { + plan_id: '2c92a00f7279a6f5017279d299d01cf9', + rate_charge_id: '2c92a0ff7279a74f017279d5bea71fc5', + name: 'storage', + price: 60, + storage: 10 + }.freeze + + LICENSE_TYPE = { + license_file: 'license file', + cloud_license: 'cloud license' + }.freeze + end + end + end +end diff --git a/qa/qa/support/matchers/eventually_matcher.rb b/qa/qa/support/matchers/eventually_matcher.rb index ff8adab424b..dedef8e6b98 100644 --- a/qa/qa/support/matchers/eventually_matcher.rb +++ b/qa/qa/support/matchers/eventually_matcher.rb @@ -59,8 +59,10 @@ module QA def wait_and_check(actual, expectation_name) attempt = 0 - QA::Runtime::Logger.debug("Running eventually matcher with '#{operator_msg}' operator") - QA::Support::Retrier.retry_until(**@retry_args) do + QA::Runtime::Logger.debug( + "Running eventually matcher with '#{operator_msg}' operator with: #{@retry_args}" + ) + QA::Support::Retrier.retry_until(**@retry_args, log: false) do QA::Runtime::Logger.debug("evaluating expectation, attempt: #{attempt += 1}") public_send(expectation_name, actual) diff --git a/qa/qa/support/matchers/have_matcher.rb b/qa/qa/support/matchers/have_matcher.rb index 7001f53a7b7..47d2d246460 100644 --- a/qa/qa/support/matchers/have_matcher.rb +++ b/qa/qa/support/matchers/have_matcher.rb @@ -19,6 +19,7 @@ module QA related_issue_item snippet_description tag + label ].each do |predicate| RSpec::Matchers.define "have_#{predicate}" do |*args, **kwargs| match do |page_object| diff --git a/qa/qa/support/repeater.rb b/qa/qa/support/repeater.rb index b3a2472d702..a4e8035f964 100644 --- a/qa/qa/support/repeater.rb +++ b/qa/qa/support/repeater.rb @@ -18,17 +18,34 @@ module QA sleep_interval: 0, raise_on_failure: true, retry_on_exception: false, - log: true + log: true, + message: nil ) attempts = 0 start = Time.now begin while remaining_attempts?(attempts, max_attempts) && remaining_time?(start, max_duration) - QA::Runtime::Logger.debug("Attempt number #{attempts + 1}") if max_attempts && log + # start logging from the second attempt + if log && attempts == 1 + msg = ["Retrying action with:"] + msg << "max_attempts: #{max_attempts};" if max_attempts + msg << "max_duration: #{max_duration};" if max_duration + msg << "reload_page: #{reload_page};" if reload_page + msg << "sleep_interval: #{sleep_interval};" + msg << "raise_on_failure: #{raise_on_failure};" + msg << "retry_on_exception: #{retry_on_exception}" + + QA::Runtime::Logger.debug(msg.join(' ')) + end + + QA::Runtime::Logger.debug("Attempt number #{attempts + 1}") if log && max_attempts && attempts > 0 result = yield - return result if result + if result + log_completion(log, attempts) + return result + end sleep_and_reload_if_needed(sleep_interval, reload_page) attempts += 1 @@ -47,13 +64,18 @@ module QA unless remaining_attempts?(attempts, max_attempts) raise( RetriesExceededError, - "Retry condition not met after #{max_attempts} #{'attempt'.pluralize(max_attempts)}" + "#{message || 'Retry'} failed after #{max_attempts} #{'attempt'.pluralize(max_attempts)}" ) end - raise WaitExceededError, "Wait condition not met after #{max_duration} #{'second'.pluralize(max_duration)}" + raise( + WaitExceededError, + "#{message || 'Wait'} failed after #{max_duration} #{'second'.pluralize(max_duration)}" + ) end + log_completion(log, attempts) + false end @@ -71,6 +93,17 @@ module QA def remaining_time?(start, max_duration) max_duration ? Time.now - start < max_duration : true end + + # Log completion if more than one attempt performed + # + # @param [Boolean] log + # @param [Integer] attempts + # @return [void] + def log_completion(log, attempts) + return unless log && attempts > 0 + + QA::Runtime::Logger.debug('ended retry') + end end end end diff --git a/qa/qa/support/retrier.rb b/qa/qa/support/retrier.rb index aa568d633fc..aa36bf5922e 100644 --- a/qa/qa/support/retrier.rb +++ b/qa/qa/support/retrier.rb @@ -7,21 +7,15 @@ module QA module_function - def retry_on_exception(max_attempts: 3, reload_page: nil, sleep_interval: 0.5, log: true) - if log - msg = ["with retry_on_exception: max_attempts: #{max_attempts}"] - msg << "reload_page: #{reload_page}" if reload_page - msg << "sleep_interval: #{sleep_interval}" - QA::Runtime::Logger.debug(msg.join('; ')) - end - + def retry_on_exception(max_attempts: 3, reload_page: nil, sleep_interval: 0.5, log: true, message: nil) result = nil repeat_until( max_attempts: max_attempts, reload_page: reload_page, sleep_interval: sleep_interval, retry_on_exception: true, - log: log + log: log, + message: message ) do result = yield @@ -29,7 +23,6 @@ module QA # We set it to `true` so that it doesn't repeat if there's no exception true end - QA::Runtime::Logger.debug("ended retry_on_exception") if log result end @@ -41,25 +34,12 @@ module QA sleep_interval: 0, raise_on_failure: true, retry_on_exception: false, - log: true + log: true, + message: nil ) # For backwards-compatibility max_attempts = 3 if max_attempts.nil? && max_duration.nil? - if log - start_msg = ["with retry_until:"] - start_msg << "max_attempts: #{max_attempts};" if max_attempts - start_msg << "max_duration: #{max_duration};" if max_duration - start_msg.push(*[ - "reload_page: #{reload_page};", - "sleep_interval: #{sleep_interval};", - "raise_on_failure: #{raise_on_failure};", - "retry_on_exception: #{retry_on_exception}" - ]) - - QA::Runtime::Logger.debug(start_msg.join(' ')) - end - result = nil repeat_until( max_attempts: max_attempts, @@ -68,11 +48,11 @@ module QA sleep_interval: sleep_interval, raise_on_failure: raise_on_failure, retry_on_exception: retry_on_exception, - log: log + log: log, + message: message ) do result = yield end - QA::Runtime::Logger.debug("ended retry_until") if log result end diff --git a/qa/qa/support/waiter.rb b/qa/qa/support/waiter.rb index 9ccc0d9484f..6dbbd197b01 100644 --- a/qa/qa/support/waiter.rb +++ b/qa/qa/support/waiter.rb @@ -7,20 +7,16 @@ module QA module_function - def wait_until(max_duration: singleton_class::DEFAULT_MAX_WAIT_TIME, reload_page: nil, sleep_interval: 0.1, raise_on_failure: true, retry_on_exception: false, log: true) - if log - QA::Runtime::Logger.debug( - <<~MSG.tr("\n", ' ') - with wait_until: max_duration: #{max_duration}; - reload_page: #{reload_page}; - sleep_interval: #{sleep_interval}; - raise_on_failure: #{raise_on_failure} - MSG - ) - end - + def wait_until( + max_duration: singleton_class::DEFAULT_MAX_WAIT_TIME, + reload_page: nil, + sleep_interval: 0.1, + raise_on_failure: true, + retry_on_exception: false, + log: true + ) result = nil - self.repeat_until( + repeat_until( max_duration: max_duration, reload_page: reload_page, sleep_interval: sleep_interval, @@ -30,7 +26,6 @@ module QA ) do result = yield end - QA::Runtime::Logger.debug("ended wait_until") if log result end diff --git a/qa/qa/tools/reliable_report.rb b/qa/qa/tools/reliable_report.rb new file mode 100644 index 00000000000..9d2079171c1 --- /dev/null +++ b/qa/qa/tools/reliable_report.rb @@ -0,0 +1,234 @@ +# frozen_string_literal: true + +require "influxdb-client" +require "terminal-table" +require "slack-notifier" + +module QA + module Tools + class ReliableReport + def initialize(run_type, range = 30) + @results = 10 + @slack_channel = "#quality-reports" + @range = range + @run_type = run_type + @stable_title = "Top #{results} stable specs for past #{@range} days in '#{run_type}' runs" + @unstable_title = "Top #{results} unstable reliable specs for past #{@range} days in '#{run_type}' runs" + end + + # Print top stable specs + # + # @return [void] + def show_top_stable + puts terminal_table( + rows: top_stable.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] }, + title: stable_title + ) + end + + # Post top stable spec report to slack + # Slice table in to multiple messages due to max char limitation + # + # @return [void] + def notify_top_stable + tables = top_stable.each_slice(5).map do |slice| + terminal_table( + rows: slice.map { |spec| [name_column(spec[0], spec[1][:file]), *table_params(spec[1].values)] } + ) + end + + puts "\nSending top stable spec report to #{slack_channel} slack channel" + slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" } + notifier.post(text: "*#{stable_title}*", **slack_args) + tables.each { |table| notifier.post(text: "```#{table}```", **slack_args) } + end + + # Print top unstable specs + # + # @return [void] + def show_top_unstable + return puts("No unstable tests present!") if top_unstable_reliable.empty? + + puts terminal_table( + rows: top_unstable_reliable.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] }, + title: unstable_title + ) + end + + # Post top unstable reliable spec report to slack + # Slice table in to multiple messages due to max char limitation + # + # @return [void] + def notify_top_unstable + return puts("No unstable tests present!") if top_unstable_reliable.empty? + + tables = top_unstable_reliable.each_slice(5).map do |slice| + terminal_table( + rows: slice.map { |spec| [name_column(spec[0], spec[1][:file]), *table_params(spec[1].values)] } + ) + end + + puts "\nSending top unstable reliable spec report to #{slack_channel} slack channel" + slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" } + notifier.post(text: "*#{unstable_title}*", **slack_args) + tables.each { |table| notifier.post(text: "```#{table}```", **slack_args) } + end + + private + + attr_reader :results, + :slack_channel, + :range, + :run_type, + :stable_title, + :unstable_title + + # Top stable specs + # + # @return [Hash] + def top_stable + @top_stable ||= runs(reliable: false).sort_by { |k, v| [v[:failure_rate], -v[:runs]] }[0..results - 1].to_h + end + + # Top unstable reliable specs + # + # @return [Hash] + def top_unstable_reliable + @top_unstable_reliable ||= runs(reliable: true) + .reject { |k, v| v[:failure_rate] == 0 } + .sort_by { |k, v| -v[:failure_rate] }[0..results - 1] + .to_h + end + + # Terminal table for result formatting + # + # @return [Terminal::Table] + def terminal_table(rows:, title: nil) + Terminal::Table.new( + headings: ["name", "runs", "failed", "failure rate"], + style: { all_separators: true }, + title: title, + rows: rows + ) + end + + # Spec parameters for table row + # + # @param [Array] parameters + # @return [Array] + def table_params(parameters) + [*parameters[1..2], "#{parameters.last}%"] + end + + # Name column value + # + # @param [String] name + # @param [String] file + # @return [String] + def name_column(name, file) + spec_name = name.length > 100 ? "#{name} ".scan(/.{1,100} /).map(&:strip).join("\n") : name + name_line = "name: '#{spec_name}'" + file_line = "file: '#{file}'" + + "#{name_line}\n#{file_line.ljust(110)}" + end + + # Test executions grouped by name + # + # @param [Boolean] reliable + # @return [Hash] + def runs(reliable:) + puts("Fetching data on #{reliable ? 'reliable ' : ''}test execution for past 30 days in '#{run_type}' runs") + puts + + query_api.query(query: query(reliable)).values.each_with_object({}) do |table, result| + records = table.records + name = records.last.values["name"] + file = records.last.values["file_path"].split("/").last + runs = records.count + failed = records.count { |r| r.values["status"] == "failed" } + failure_rate = (failed.to_f / runs.to_f) * 100 + + result[name] = { + file: file, + runs: runs, + failed: failed, + failure_rate: failure_rate == 0 ? failure_rate.round(0) : failure_rate.round(2) + } + end + end + + # Flux query + # + # @param [Boolean] reliable + # @return [String] + def query(reliable) + <<~QUERY + from(bucket: "e2e-test-stats") + |> range(start: -#{range}d) + |> filter(fn: (r) => r._measurement == "test-stats" and + r.run_type == "#{run_type}" and + r.status != "pending" and + r.merge_request == "false" and + r.quarantined == "false" and + r.reliable == "#{reliable}" and + r._field == "id" + ) + |> group(columns: ["name"]) + QUERY + end + + # Query client + # + # @return [QueryApi] + def query_api + @query_api ||= influx_client.create_query_api + end + + # InfluxDb client + # + # @return [InfluxDB2::Client] + def influx_client + @influx_client ||= InfluxDB2::Client.new( + influxdb_url, + influxdb_token, + bucket: "e2e-test-stats", + org: "gitlab-qa", + precision: InfluxDB2::WritePrecision::NANOSECOND + ) + end + + # Slack notifier + # + # @return [Slack::Notifier] + def notifier + @notifier ||= Slack::Notifier.new( + slack_webhook_url, + channel: slack_channel, + username: "Reliable spec reporter" + ) + end + + # InfluxDb instance url + # + # @return [String] + def influxdb_url + @influxdb_url ||= ENV["QA_INFLUXDB_URL"] || raise("Missing QA_INFLUXDB_URL environment variable") + end + + # Influxdb token + # + # @return [String] + def influxdb_token + @influxdb_token ||= ENV["QA_INFLUXDB_TOKEN"] || raise("Missing QA_INFLUXDB_TOKEN environment variable") + end + + # Slack webhook url + # + # @return [String] + def slack_webhook_url + @slack_webhook_url ||= ENV["CI_SLACK_WEBHOOK_URL"] || raise("Missing CI_SLACK_WEBHOOK_URL environment variable") + end + end + end +end diff --git a/qa/spec/runtime/feature_spec.rb b/qa/spec/runtime/feature_spec.rb index 39c20dd3070..88f5cd5be93 100644 --- a/qa/spec/runtime/feature_spec.rb +++ b/qa/spec/runtime/feature_spec.rb @@ -175,6 +175,20 @@ RSpec.describe QA::Runtime::Feature do expect(described_class.enabled?(feature_flag)).to be_truthy end + it 'raises an error when the scope is unknown' do + expect(QA::Runtime::API::Request) + .to receive(:new) + .with(api_client, "/features") + .and_return(request) + expect(described_class) + .to receive(:get) + .and_return( + Struct.new(:code, :body) + .new(200, %([{ "name": "a_flag", "state": "conditional", "gates": { "key": "groups", "value": ["foo"] } }]))) + + expect { described_class.enabled?(feature_flag, scope: 'foo') }.to raise_error(QA::Runtime::Feature::UnknownScopeError) + end + context 'when a project scope is provided' do it_behaves_like 'checks a feature flag' do let(:scope) { :project } @@ -212,4 +226,38 @@ RSpec.describe QA::Runtime::Feature do end end end + + describe '.set' do + let(:scope) { { scope: 'actor' } } + + it 'raises an error when the flag state is unknown' do + expect(described_class).not_to receive(:enable) + expect(described_class).not_to receive(:disable) + + expect { described_class.set({ foo: 'bar' }, **scope) }.to raise_error(QA::Runtime::Feature::UnknownStateError, 'Unknown feature flag state: bar') + end + + it 'enables feature flags' do + expect(described_class).to receive(:enable).with(:flag1, scope) + expect(described_class).to receive(:enable).with(:flag2, scope) + expect(described_class).not_to receive(:disable) + + described_class.set({ flag1: 'enabled', flag2: 'enable' }, **scope) + end + + it 'disables feature flags' do + expect(described_class).to receive(:disable).with(:flag1, scope) + expect(described_class).to receive(:disable).with(:flag2, scope) + expect(described_class).not_to receive(:enable) + + described_class.set({ flag1: 'disable', flag2: 'disable' }, **scope) + end + + it 'enables and disables feature flags' do + expect(described_class).to receive(:enable).with(:flag1, scope) + expect(described_class).to receive(:disable).with(:flag2, scope) + + described_class.set({ flag1: 'enabled', flag2: 'disabled' }, **scope) + end + end end diff --git a/qa/spec/scenario/test/instance/reliable_spec.rb b/qa/spec/scenario/test/instance/reliable_spec.rb new file mode 100644 index 00000000000..4001d386bf3 --- /dev/null +++ b/qa/spec/scenario/test/instance/reliable_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +RSpec.describe QA::Scenario::Test::Instance::Reliable do + it_behaves_like 'a QA scenario class' do + let(:tags) { [:reliable] } + end +end diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb index e25892a008f..640f2de0ca2 100644 --- a/qa/spec/spec_helper.rb +++ b/qa/spec/spec_helper.rb @@ -27,8 +27,12 @@ RSpec.configure do |config| config.add_formatter QA::Support::Formatters::QuarantineFormatter config.add_formatter QA::Support::Formatters::TestStatsFormatter if QA::Runtime::Env.export_metrics? - config.before do |example| + config.prepend_before do |example| QA::Runtime::Logger.debug("\nStarting test: #{example.full_description}\n") + + # Reset fabrication counters tracked in resource base + Thread.current[:api_fabrication] = 0 + Thread.current[:browser_ui_fabrication] = 0 end config.after do @@ -36,6 +40,12 @@ RSpec.configure do |config| QA::Git::Repository.new.delete_netrc end + # Add fabrication time to spec metadata + config.append_after do |example| + example.metadata[:api_fabrication] = Thread.current[:api_fabrication] + example.metadata[:browser_ui_fabrication] = Thread.current[:browser_ui_fabrication] + end + config.after(:context) do if !QA::Runtime::Browser.blank_page? && QA::Page::Main::Menu.perform(&:signed_in?) QA::Page::Main::Menu.perform(&:sign_out) diff --git a/qa/spec/specs/allure_report_spec.rb b/qa/spec/specs/allure_report_spec.rb index 03bf77039cc..06b09106140 100644 --- a/qa/spec/specs/allure_report_spec.rb +++ b/qa/spec/specs/allure_report_spec.rb @@ -3,7 +3,7 @@ describe QA::Runtime::AllureReport do include QA::Support::Helpers::StubEnv - let(:rspec_config) { double('RSpec::Core::Configuration', 'add_formatter': nil, after: nil) } + let(:rspec_config) { double('RSpec::Core::Configuration', 'add_formatter': nil, append_after: nil) } let(:png_path) { 'png_path' } let(:html_path) { 'html_path' } @@ -46,6 +46,8 @@ describe QA::Runtime::AllureReport do let(:html_file) { 'html-file' } let(:ci_job) { 'ee:relative 5' } let(:versions) { { version: '14', revision: '6ced31db947' } } + let(:session) { double('session') } + let(:browser_log) { ['log message 1', 'log message 2'] } before do stub_env('CI', 'true') @@ -58,6 +60,9 @@ describe QA::Runtime::AllureReport do allow(RestClient::Request).to receive(:execute) { double('response', code: 200, body: versions.to_json) } allow(QA::Runtime::Scenario).to receive(:method_missing).with(:gitlab_address).and_return('gitlab.com') + allow(Capybara).to receive(:current_session).and_return(session) + allow(session).to receive_message_chain('driver.browser.logs.get').and_return(browser_log) + described_class.configure! end @@ -76,7 +81,11 @@ describe QA::Runtime::AllureReport do .with(QA::Support::Formatters::AllureMetadataFormatter).ordered end - it 'configures screenshot saving' do + it 'configures attachments saving' do + expect(rspec_config).to have_received(:append_after) do |&arg| + arg.call + end + aggregate_failures do expect(Allure).to have_received(:add_attachment).with( name: 'screenshot', @@ -90,6 +99,12 @@ describe QA::Runtime::AllureReport do type: 'text/html', test_case: true ) + expect(Allure).to have_received(:add_attachment).with( + name: 'browser.log', + source: browser_log.join("\n\n"), + type: Allure::ContentType::TXT, + test_case: true + ) end end end diff --git a/qa/spec/specs/helpers/context_selector_spec.rb b/qa/spec/specs/helpers/context_selector_spec.rb index 0152fee6f5b..5a320cde71f 100644 --- a/qa/spec/specs/helpers/context_selector_spec.rb +++ b/qa/spec/specs/helpers/context_selector_spec.rb @@ -186,6 +186,24 @@ RSpec.describe QA::Specs::Helpers::ContextSelector do end end + context 'staging-ref' do + before do + QA::Runtime::Scenario.define(:gitlab_address, 'https://staging-ref.gitlab.com/') + end + + it 'runs on staging-ref' do + group = describe_successfully do + it('does not run in staging', only: { subdomain: :staging }) {} + it('runs in staging-ref', only: { subdomain: /^staging-ref./ }) {} + end + + aggregate_failures do + expect(group.examples[0].execution_result.status).to eq(:pending) + expect(group.examples[1].execution_result.status).to eq(:passed) + end + end + end + context 'production' do before do QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.com/') diff --git a/qa/spec/support/formatters/test_stats_formatter_spec.rb b/qa/spec/support/formatters/test_stats_formatter_spec.rb index 859d45a660b..f9baf9bd9d9 100644 --- a/qa/spec/support/formatters/test_stats_formatter_spec.rb +++ b/qa/spec/support/formatters/test_stats_formatter_spec.rb @@ -20,6 +20,8 @@ describe QA::Support::Formatters::TestStatsFormatter do let(:influx_write_api) { instance_double('InfluxDB2::WriteApi', write: nil) } let(:stage) { '1_manage' } let(:file_path) { "./qa/specs/features/#{stage}/subfolder/some_spec.rb" } + let(:ui_fabrication) { 0 } + let(:api_fabrication) { 0 } let(:influx_client_args) do { @@ -48,6 +50,9 @@ describe QA::Support::Formatters::TestStatsFormatter do fields: { id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]', run_time: 0, + api_fabrication: api_fabrication * 1000, + ui_fabrication: ui_fabrication * 1000, + total_fabrication: (api_fabrication + ui_fabrication) * 1000, retry_attempts: 0, job_url: ci_job_url, pipeline_url: ci_pipeline_url, @@ -69,6 +74,11 @@ describe QA::Support::Formatters::TestStatsFormatter do RSpec::Core::Sandbox.sandboxed do |config| config.formatter = QA::Support::Formatters::TestStatsFormatter + config.append_after do |example| + example.metadata[:api_fabrication] = Thread.current[:api_fabrication] + example.metadata[:browser_ui_fabrication] = Thread.current[:browser_ui_fabrication] + end + config.before(:context) { RSpec.current_example = nil } example.run @@ -171,5 +181,21 @@ describe QA::Support::Formatters::TestStatsFormatter do expect(influx_write_api).to have_received(:write).with(data: [data]) end end + + context 'with fabrication runtimes' do + let(:ui_fabrication) { 10 } + let(:api_fabrication) { 4 } + + before do + Thread.current[:api_fabrication] = api_fabrication + Thread.current[:browser_ui_fabrication] = ui_fabrication + end + + it 'exports data to influxdb with fabrication times' do + run_spec + + expect(influx_write_api).to have_received(:write).with(data: [data]) + end + end end end diff --git a/qa/spec/support/repeater_spec.rb b/qa/spec/support/repeater_spec.rb index da8d6b18fb0..4fa3bcde5e7 100644 --- a/qa/spec/support/repeater_spec.rb +++ b/qa/spec/support/repeater_spec.rb @@ -23,7 +23,7 @@ RSpec.describe QA::Support::Repeater do context 'when retry_on_exception is not provided (default: false)' do context 'when max_duration is provided' do context 'when max duration is reached' do - it 'raises an exception' do + it 'raises an exception with default message' do expect do Timecop.freeze do subject.repeat_until(max_duration: 1) do @@ -31,7 +31,20 @@ RSpec.describe QA::Support::Repeater do false end end - end.to raise_error(QA::Support::Repeater::WaitExceededError, "Wait condition not met after 1 second") + end.to raise_error(QA::Support::Repeater::WaitExceededError, "Wait failed after 1 second") + end + + it 'raises an exception with custom message' do + message = 'Some custom action' + + expect do + Timecop.freeze do + subject.repeat_until(max_duration: 1, message: message) do + Timecop.travel(2) + false + end + end + end.to raise_error(QA::Support::Repeater::WaitExceededError, "#{message} failed after 1 second") end it 'ignores attempts' do @@ -70,14 +83,26 @@ RSpec.describe QA::Support::Repeater do context 'when max_attempts is provided' do context 'when max_attempts is reached' do - it 'raises an exception' do + it 'raises an exception with default message' do expect do Timecop.freeze do subject.repeat_until(max_attempts: 1) do false end end - end.to raise_error(QA::Support::Repeater::RetriesExceededError, "Retry condition not met after 1 attempt") + end.to raise_error(QA::Support::Repeater::RetriesExceededError, "Retry failed after 1 attempt") + end + + it 'raises an exception with custom message' do + message = 'Some custom action' + + expect do + Timecop.freeze do + subject.repeat_until(max_attempts: 1, message: message) do + false + end + end + end.to raise_error(QA::Support::Repeater::RetriesExceededError, "#{message} failed after 1 attempt") end it 'ignores duration' do @@ -126,7 +151,7 @@ RSpec.describe QA::Support::Repeater do false end end - end.to raise_error(QA::Support::Repeater::RetriesExceededError, "Retry condition not met after 1 attempt") + end.to raise_error(QA::Support::Repeater::RetriesExceededError, "Retry failed after 1 attempt") end end @@ -141,7 +166,7 @@ RSpec.describe QA::Support::Repeater do false end end - end.to raise_error(QA::Support::Repeater::WaitExceededError, "Wait condition not met after 1 second") + end.to raise_error(QA::Support::Repeater::WaitExceededError, "Wait failed after 1 second") end end end @@ -210,7 +235,7 @@ RSpec.describe QA::Support::Repeater do false end end - end.to raise_error(QA::Support::Repeater::RetriesExceededError, "Retry condition not met after 1 attempt") + end.to raise_error(QA::Support::Repeater::RetriesExceededError, "Retry failed after 1 attempt") end end @@ -225,7 +250,7 @@ RSpec.describe QA::Support::Repeater do false end end - end.to raise_error(QA::Support::Repeater::WaitExceededError, "Wait condition not met after 1 second") + end.to raise_error(QA::Support::Repeater::WaitExceededError, "Wait failed after 1 second") end end end @@ -380,34 +405,67 @@ RSpec.describe QA::Support::Repeater do end end - it 'logs attempts' do - attempted = false + context 'with logging' do + before do + allow(QA::Runtime::Logger).to receive(:debug) + end - expect do - subject.repeat_until(max_attempts: 1) do - unless attempted - attempted = true - break false - end + it 'skips logging single attempt with max_attempts' do + subject.repeat_until(max_attempts: 3) do + true + end + expect(QA::Runtime::Logger).not_to have_received(:debug) + end + + it 'skips logging single attempt with max_duration' do + subject.repeat_until(max_duration: 3) do true end - end.to output(/Attempt number/).to_stdout_from_any_process - end - it 'allows logging to be silenced' do - attempted = false + expect(QA::Runtime::Logger).not_to have_received(:debug) + end - expect do - subject.repeat_until(max_attempts: 1, log: false) do - unless attempted - attempted = true - break false - end + it 'allows logging to be silenced' do + subject.repeat_until(max_attempts: 3, log: false, raise_on_failure: false) do + false + end - true + expect(QA::Runtime::Logger).not_to have_received(:debug) + end + + it 'starts logging on subsequent attempts for max_duration' do + subject.repeat_until(max_duration: 0.3, sleep_interval: 0.1, raise_on_failure: false) do + false + end + + aggregate_failures do + expect(QA::Runtime::Logger).to have_received(:debug).with(<<~MSG.strip).ordered.once + Retrying action with: max_duration: 0.3; sleep_interval: 0.1; raise_on_failure: false; retry_on_exception: false + MSG + expect(QA::Runtime::Logger).to have_received(:debug).with('ended retry').ordered.once + expect(QA::Runtime::Logger).not_to have_received(:debug).with(/Attempt number/) + end + end + + it 'starts logging subsequent attempts for max_attempts' do + attempts = 0 + subject.repeat_until(max_attempts: 4, raise_on_failure: false) do + next true if attempts == 2 + + attempts += 1 + false end - end.not_to output.to_stdout_from_any_process + + aggregate_failures do + expect(QA::Runtime::Logger).to have_received(:debug).with(<<~MSG.strip).ordered.once + Retrying action with: max_attempts: 4; sleep_interval: 0; raise_on_failure: false; retry_on_exception: false + MSG + expect(QA::Runtime::Logger).to have_received(:debug).with('Attempt number 2').ordered.once + expect(QA::Runtime::Logger).to have_received(:debug).with('Attempt number 3').ordered.once + expect(QA::Runtime::Logger).to have_received(:debug).with('ended retry').ordered.once + end + end end end end diff --git a/qa/spec/support/retrier_spec.rb b/qa/spec/support/retrier_spec.rb index 9ad3e85fea9..1f303093a00 100644 --- a/qa/spec/support/retrier_spec.rb +++ b/qa/spec/support/retrier_spec.rb @@ -1,42 +1,7 @@ # frozen_string_literal: true RSpec.describe QA::Support::Retrier do - before do - logger = ::Logger.new $stdout - logger.level = ::Logger::DEBUG - QA::Runtime::Logger.logger = logger - end - describe '.retry_until' do - context 'when the condition is true' do - it 'logs max attempts (3 by default)' do - expect { subject.retry_until { true } } - .to output(/with retry_until: max_attempts: 3; reload_page: ; sleep_interval: 0; raise_on_failure: true; retry_on_exception: false/).to_stdout_from_any_process - end - - it 'logs max duration' do - expect { subject.retry_until(max_duration: 1) { true } } - .to output(/with retry_until: max_duration: 1; reload_page: ; sleep_interval: 0; raise_on_failure: true; retry_on_exception: false/).to_stdout_from_any_process - end - - it 'logs the end' do - expect { subject.retry_until { true } } - .to output(/ended retry_until$/).to_stdout_from_any_process - end - end - - context 'when the condition is false' do - it 'logs the start' do - expect { subject.retry_until(max_duration: 0, raise_on_failure: false) { false } } - .to output(/with retry_until: max_duration: 0; reload_page: ; sleep_interval: 0; raise_on_failure: false; retry_on_exception: false/).to_stdout_from_any_process - end - - it 'logs the end' do - expect { subject.retry_until(max_duration: 0, raise_on_failure: false) { false } } - .to output(/ended retry_until$/).to_stdout_from_any_process - end - end - context 'when max_duration and max_attempts are nil' do it 'sets max attempts to 3 by default' do expect(subject).to receive(:repeat_until).with(hash_including(max_attempts: 3)) @@ -62,35 +27,21 @@ RSpec.describe QA::Support::Retrier do subject.retry_until end - end - describe '.retry_on_exception' do - context 'when the condition is true' do - it 'logs max_attempts, reload_page, and sleep_interval parameters' do - message = /with retry_on_exception: max_attempts: 1; reload_page: true; sleep_interval: 0/ - expect { subject.retry_on_exception(max_attempts: 1, reload_page: true, sleep_interval: 0) { true } } - .to output(message).to_stdout_from_any_process - end + it 'allows logs to be silenced' do + expect(subject).to receive(:repeat_until).with(hash_including(log: false)) - it 'logs the end' do - expect { subject.retry_on_exception(max_attempts: 1, reload_page: nil, sleep_interval: 0) { true } } - .to output(/ended retry_on_exception$/).to_stdout_from_any_process - end + subject.retry_until(log: false) end - context 'when the condition is false' do - it 'logs the start' do - message = /with retry_on_exception: max_attempts: 1; reload_page: true; sleep_interval: 0/ - expect { subject.retry_on_exception(max_attempts: 1, reload_page: true, sleep_interval: 0) { false } } - .to output(message).to_stdout_from_any_process - end + it 'sets custom error message' do + expect(subject).to receive(:repeat_until).with(hash_including(message: 'Custom message')) - it 'logs the end' do - expect { subject.retry_on_exception(max_attempts: 1, reload_page: nil, sleep_interval: 0) { false } } - .to output(/ended retry_on_exception$/).to_stdout_from_any_process - end + subject.retry_until(message: 'Custom message') end + end + describe '.retry_on_exception' do it 'does not repeat if no exception is raised' do loop_counter = 0 return_value = "test passed" @@ -121,5 +72,11 @@ RSpec.describe QA::Support::Retrier do subject.retry_on_exception end + + it 'allows logs to be silenced' do + expect(subject).to receive(:repeat_until).with(hash_including(log: false)) + + subject.retry_on_exception(log: false) + end end end diff --git a/qa/spec/support/shared_contexts/packages_registry_shared_context.rb b/qa/spec/support/shared_contexts/packages_registry_shared_context.rb index 6e197015640..e686d254a44 100644 --- a/qa/spec/support/shared_contexts/packages_registry_shared_context.rb +++ b/qa/spec/support/shared_contexts/packages_registry_shared_context.rb @@ -43,8 +43,13 @@ module QA let(:project_deploy_token) do Resource::DeployToken.fabricate_via_browser_ui! do |deploy_token| - deploy_token.name = 'helm-package-deploy-token' + deploy_token.name = 'package-deploy-token' deploy_token.project = package_project + deploy_token.scopes = [ + :read_repository, + :read_package_registry, + :write_package_registry + ] end end diff --git a/qa/spec/support/waiter_spec.rb b/qa/spec/support/waiter_spec.rb index d0b216b5dc1..c575a27bc35 100644 --- a/qa/spec/support/waiter_spec.rb +++ b/qa/spec/support/waiter_spec.rb @@ -1,40 +1,11 @@ # frozen_string_literal: true RSpec.describe QA::Support::Waiter do - before do - logger = ::Logger.new $stdout - logger.level = ::Logger::DEBUG - QA::Runtime::Logger.logger = logger - end - describe '.wait_until' do - context 'when the condition is true' do - it 'logs the start' do - expect { subject.wait_until(max_duration: 0, raise_on_failure: false) { true } } - .to output(/with wait_until: max_duration: 0; reload_page: ; sleep_interval: 0.1/).to_stdout_from_any_process - end - - it 'logs the end' do - expect { subject.wait_until(max_duration: 0, raise_on_failure: false) { true } } - .to output(/ended wait_until$/).to_stdout_from_any_process - end - end - - context 'when the condition is false' do - it 'logs the start' do - expect { subject.wait_until(max_duration: 0, raise_on_failure: false) { false } } - .to output(/with wait_until: max_duration: 0; reload_page: ; sleep_interval: 0.1/).to_stdout_from_any_process - end - - it 'logs the end' do - expect { subject.wait_until(max_duration: 0, raise_on_failure: false) { false } } - .to output(/ended wait_until$/).to_stdout_from_any_process - end - end - it 'allows logs to be silenced' do - expect { subject.wait_until(max_duration: 0, raise_on_failure: false, log: false) { false } } - .not_to output.to_stdout_from_any_process + expect(subject).to receive(:repeat_until).with(hash_including(log: false)) + + subject.wait_until(log: false) end it 'sets max_duration to 60 by default' do diff --git a/qa/spec/tools/reliable_report_spec.rb b/qa/spec/tools/reliable_report_spec.rb new file mode 100644 index 00000000000..c7d4d28fb21 --- /dev/null +++ b/qa/spec/tools/reliable_report_spec.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true + +describe QA::Tools::ReliableReport do + include QA::Support::Helpers::StubEnv + + subject(:reporter) { described_class.new(run_type, range) } + + let(:slack_notifier) { instance_double("Slack::Notifier", post: nil) } + let(:influx_client) { instance_double("InfluxDB2::Client", create_query_api: query_api) } + let(:query_api) { instance_double("InfluxDB2::QueryApi") } + + let(:slack_channel) { "#quality-reports" } + let(:run_type) { "package-and-qa" } + let(:range) { 30 } + let(:results) { 10 } + + let(:runs) { { 0 => stable_spec, 1 => unstable_spec } } + + let(:stable_spec) do + spec_values = { "name" => "stable spec", "status" => "passed", "file_path" => "some/spec.rb" } + instance_double( + "InfluxDB2::FluxTable", + records: [ + instance_double("InfluxDB2::FluxRecord", values: spec_values), + instance_double("InfluxDB2::FluxRecord", values: spec_values), + instance_double("InfluxDB2::FluxRecord", values: spec_values) + ] + ) + end + + let(:unstable_spec) do + spec_values = { "name" => "unstable spec", "status" => "failed", "file_path" => "some/spec.rb" } + instance_double( + "InfluxDB2::FluxTable", + records: [ + instance_double("InfluxDB2::FluxRecord", values: { **spec_values, "status" => "passed" }), + instance_double("InfluxDB2::FluxRecord", values: spec_values), + instance_double("InfluxDB2::FluxRecord", values: spec_values) + ] + ) + end + + def flux_query(reliable) + <<~QUERY + from(bucket: "e2e-test-stats") + |> range(start: -#{range}d) + |> filter(fn: (r) => r._measurement == "test-stats" and + r.run_type == "#{run_type}" and + r.status != "pending" and + r.merge_request == "false" and + r.quarantined == "false" and + r.reliable == "#{reliable}" and + r._field == "id" + ) + |> group(columns: ["name"]) + QUERY + end + + def table(rows, title = nil) + Terminal::Table.new( + headings: ["name", "runs", "failed", "failure rate"], + style: { all_separators: true }, + title: title, + rows: rows + ) + end + + def name_column(spec_name) + name = "name: '#{spec_name}'" + file = "file: 'spec.rb'".ljust(110) + + "#{name}\n#{file}" + end + + before do + stub_env("QA_INFLUXDB_URL", "url") + stub_env("QA_INFLUXDB_TOKEN", "token") + stub_env("CI_SLACK_WEBHOOK_URL", "slack_url") + + allow(Slack::Notifier).to receive(:new).and_return(slack_notifier) + allow(InfluxDB2::Client).to receive(:new).and_return(influx_client) + allow(query_api).to receive(:query).with(query: query).and_return(runs) + end + + context "with stable spec report" do + let(:query) { flux_query(false) } + let(:fetch_message) { "Fetching data on test execution for past #{range} days in '#{run_type}' runs" } + let(:slack_send_message) { "Sending top stable spec report to #{slack_channel} slack channel" } + let(:title) { "Top #{results} stable specs for past #{range} days in '#{run_type}' runs" } + let(:rows) do + [ + [name_column("stable spec"), 3, 0, "0%"], + [name_column("unstable spec"), 3, 2, "66.67%"] + ] + end + + it "prints top stable spec report to console" do + expect { reporter.show_top_stable }.to output("#{fetch_message}\n\n#{table(rows, title)}\n").to_stdout + end + + it "sends top stable spec report to slack" do + slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" } + + expect { reporter.notify_top_stable }.to output("#{fetch_message}\n\n\n#{slack_send_message}\n").to_stdout + expect(slack_notifier).to have_received(:post).with(text: "*#{title}*", **slack_args) + expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows)}```", **slack_args) + end + end + + context "with unstable spec report" do + let(:query) { flux_query(true) } + let(:fetch_message) { "Fetching data on reliable test execution for past #{range} days in '#{run_type}' runs" } + let(:slack_send_message) { "Sending top unstable reliable spec report to #{slack_channel} slack channel" } + let(:title) { "Top #{results} unstable reliable specs for past #{range} days in '#{run_type}' runs" } + let(:rows) { [[name_column("unstable spec"), 3, 2, "66.67%"]] } + + it "prints top unstable spec report to console" do + expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{table(rows, title)}\n").to_stdout + end + + it "sends top unstable reliable spec report to slack" do + slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" } + + expect { reporter.notify_top_unstable }.to output("#{fetch_message}\n\n\n#{slack_send_message}\n").to_stdout + expect(slack_notifier).to have_received(:post).with(text: "*#{title}*", **slack_args) + expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows)}```", **slack_args) + end + end + + context "without unstable reliable specs" do + let(:query) { flux_query(true) } + let(:runs) { { 0 => stable_spec } } + let(:fetch_message) { "Fetching data on reliable test execution for past #{range} days in '#{run_type}' runs" } + let(:no_result_message) { "No unstable tests present!" } + + it "prints no result message to console" do + expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{no_result_message}\n").to_stdout + end + + it "skips slack notification" do + expect { reporter.notify_top_unstable }.to output("#{fetch_message}\n\n#{no_result_message}\n").to_stdout + expect(slack_notifier).not_to have_received(:post) + end + end +end diff --git a/qa/tasks/reliable_report.rake b/qa/tasks/reliable_report.rake new file mode 100644 index 00000000000..204c959093a --- /dev/null +++ b/qa/tasks/reliable_report.rake @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Rails/RakeEnvironment + +require_relative "../qa/tools/reliable_report" + +desc "Fetch top most reliable specs" +task :reliable_spec_report, [:run_type, :range, :create_slack_report] do |_task, args| + report = QA::Tools::ReliableReport.new(args[:run_type] || "package-and-qa", args[:range]) + + report.show_top_stable + report.notify_top_stable if args[:create_slack_report] == 'true' +end + +desc "Fetch top most unstable reliable specs" +task :unreliable_spec_report, [:run_type, :range, :create_slack_report] do |_task, args| + report = QA::Tools::ReliableReport.new(args[:run_type] || "package-and-qa", args[:range]) + + report.show_top_unstable + report.notify_top_unstable if args[:create_slack_report] == 'true' +end +# rubocop:enable Rails/RakeEnvironment |