summaryrefslogtreecommitdiff
path: root/qa/qa/tools
diff options
context:
space:
mode:
Diffstat (limited to 'qa/qa/tools')
-rw-r--r--qa/qa/tools/ci/ff_changes.rb66
-rw-r--r--qa/qa/tools/ci/helpers.rb50
-rw-r--r--qa/qa/tools/ci/non_empty_suites.rb98
-rw-r--r--qa/qa/tools/ci/qa_changes.rb116
-rw-r--r--qa/qa/tools/ci/test_results.rb78
-rw-r--r--qa/qa/tools/revoke_all_personal_access_tokens.rb44
-rw-r--r--qa/qa/tools/revoke_user_personal_access_tokens.rb94
-rw-r--r--qa/qa/tools/test_resources_handler.rb1
8 files changed, 502 insertions, 45 deletions
diff --git a/qa/qa/tools/ci/ff_changes.rb b/qa/qa/tools/ci/ff_changes.rb
new file mode 100644
index 00000000000..67e52633833
--- /dev/null
+++ b/qa/qa/tools/ci/ff_changes.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module QA
+ module Tools
+ module Ci
+ class FfChanges
+ include Helpers
+
+ def initialize(mr_diff)
+ @mr_diff = mr_diff
+ end
+
+ # Return list of feature flags changed in mr with inverse or deleted state
+ #
+ # @return [String]
+ def fetch
+ logger.info("Detecting feature flag changes")
+ ff_toggles = mr_diff.map do |change|
+ ff_yaml = ff_yaml_for_file(change)
+ next unless ff_yaml
+
+ state = if ff_yaml[:deleted]
+ "deleted"
+ else
+ ff_yaml[:default_enabled] ? 'disabled' : 'enabled'
+ end
+
+ logger.info(" found changes in feature flag '#{ff_yaml[:name]}'")
+ "#{ff_yaml[:name]}=#{state}"
+ end.compact
+
+ if ff_toggles.empty?
+ logger.info(" no changes to feature flags detected, skipping!")
+ return
+ end
+
+ logger.info(" constructed feature flag states: '#{ff_toggles}'")
+ ff_toggles.join(",")
+ end
+
+ private
+
+ attr_reader :mr_diff
+
+ # Loads the YAML feature flag definition based on changed files in merge requests.
+ # The definition is loaded from the definition file itself.
+ #
+ # @param [Hash] change mr file change
+ # @return [Hash] a hash containing the YAML data for the feature flag definition
+ def ff_yaml_for_file(change)
+ return unless change[:path] =~ %r{/feature_flags/(development|ops)/.*\.yml}
+ if change[:deleted_file]
+ return { name: change[:path].split("/").last.gsub(/\.(yml|yaml)/, ""), deleted: true }
+ end
+
+ YAML.safe_load(
+ File.read(File.expand_path("../#{change[:path]}", QA::Runtime::Path.qa_root)),
+ symbolize_names: true
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/qa/qa/tools/ci/helpers.rb b/qa/qa/tools/ci/helpers.rb
new file mode 100644
index 00000000000..55bb123de20
--- /dev/null
+++ b/qa/qa/tools/ci/helpers.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module QA
+ module Tools
+ module Ci
+ # Helpers for CI related tasks
+ #
+ module Helpers
+ include Support::API
+
+ # Logger instance
+ #
+ # @return [Logger]
+ def logger
+ @logger ||= Gitlab::QA::TestLogger.logger(
+ level: Gitlab::QA::Runtime::Env.log_level,
+ source: "CI Tools"
+ )
+ end
+
+ # Api get request
+ #
+ # @param [String] path
+ # @param [Hash] args
+ # @return [Hash, Array]
+ def api_get(path, **args)
+ response = get("#{api_url}/#{path}", { headers: { "PRIVATE-TOKEN" => access_token }, **args })
+ response = response.follow_redirection if response.code == Support::API::HTTP_STATUS_PERMANENT_REDIRECT
+ raise "Request failed: '#{response.body}'" unless response.code == Support::API::HTTP_STATUS_OK
+
+ args[:raw_response] ? response : parse_body(response)
+ end
+
+ # Gitlab api url
+ #
+ # @return [String]
+ def api_url
+ @api_url ||= ENV.fetch('CI_API_V4_URL', 'https://gitlab.com/api/v4')
+ end
+
+ # Api access token
+ #
+ # @return [String]
+ def access_token
+ @access_token ||= ENV.fetch('QA_GITLAB_CI_TOKEN') { raise('Variable QA_GITLAB_CI_TOKEN missing') }
+ end
+ end
+ end
+ end
+end
diff --git a/qa/qa/tools/ci/non_empty_suites.rb b/qa/qa/tools/ci/non_empty_suites.rb
new file mode 100644
index 00000000000..687c11a3e62
--- /dev/null
+++ b/qa/qa/tools/ci/non_empty_suites.rb
@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module QA
+ module Tools
+ module Ci
+ # Run count commands for scenarios and detect which ones have more than 0 examples to run
+ #
+ class NonEmptySuites
+ include Helpers
+
+ # rubocop:disable Layout/LineLength
+ SCENARIOS = [
+ { klass: "Test::Instance::All" },
+ { klass: "Test::Instance::Smoke" },
+ { klass: "Test::Instance::Reliable" },
+ { klass: "Test::Instance::ReviewBlocking" },
+ { klass: "Test::Instance::ReviewNonBlocking" },
+ { klass: "Test::Instance::CloudActivation" },
+ { klass: "Test::Instance::Integrations" },
+ { klass: "Test::Instance::Jira" },
+ { klass: "Test::Instance::LargeSetup" },
+ { klass: "Test::Instance::Metrics" },
+ { klass: "Test::Instance::ObjectStorage" },
+ { klass: "Test::Instance::Packages" },
+ { klass: "Test::Instance::RepositoryStorage" },
+ { klass: "Test::Integration::ServicePingDisabled" },
+ { klass: "Test::Integration::LDAPNoTLS" },
+ { klass: "Test::Integration::LDAPTLS" },
+ { klass: "Test::Integration::LDAPNoServer" },
+ { klass: "Test::Integration::InstanceSAML" },
+ { klass: "Test::Integration::RegistryWithCDN" },
+ { klass: "Test::Integration::RegistryTLS" },
+ { klass: "Test::Integration::Registry" },
+ { klass: "Test::Integration::SMTP" },
+ { klass: "QA::EE::Scenario::Test::Integration::Elasticsearch" },
+ { klass: "QA::EE::Scenario::Test::Integration::GroupSAML" },
+ {
+ klass: "QA::EE::Scenario::Test::Geo",
+ args: "--primary-address http://dummy1.test --primary-name gitlab-primary --secondary-address http://dummy2.test --secondary-name gitlab-secondary --without-setup"
+ },
+ {
+ klass: "Test::Integration::Mattermost",
+ args: "--mattermost-address http://mattermost.test"
+ }
+ ].freeze
+ # rubocop:enable Layout/LineLength
+
+ def initialize(qa_tests)
+ @qa_tests = qa_tests
+ end
+
+ # Run counts and return runnable scenario list
+ #
+ # @return [String]
+ def fetch
+ logger.info("Checking for runnable suites")
+ scenarios = SCENARIOS.each_with_object([]) do |scenario, runnable_scenarios|
+ logger.info(" fetching runnable specs for '#{scenario[:klass]}'")
+
+ out, err, status = run_command(**scenario)
+
+ unless status.success?
+ logger.error(" example count failed!\n#{err}")
+ next
+ end
+
+ count = out.split("\n").last.to_i
+ logger.info(" found #{count} examples to run")
+ runnable_scenarios << scenario[:klass] if count > 0
+ end
+
+ scenarios.join(",")
+ end
+
+ private
+
+ attr_reader :qa_tests
+
+ # Run scenario count command
+ #
+ # @param [String] klass
+ # @param [String] args
+ # @return [String]
+ def run_command(klass:, args: nil)
+ cmd = ["bundle exec bin/qa"]
+ cmd << klass
+ cmd << "--count-examples-only --address http://dummy1.test"
+ cmd << args if args
+ cmd << "-- #{qa_tests}" unless qa_tests.blank?
+
+ Open3.capture3(cmd.join(" "))
+ end
+ end
+ end
+ end
+end
diff --git a/qa/qa/tools/ci/qa_changes.rb b/qa/qa/tools/ci/qa_changes.rb
new file mode 100644
index 00000000000..75274961efe
--- /dev/null
+++ b/qa/qa/tools/ci/qa_changes.rb
@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module QA
+ module Tools
+ module Ci
+ # Determine specific qa specs or paths to execute based on changes
+ class QaChanges
+ include Helpers
+
+ QA_PATTERN = %r{^qa/}.freeze
+ SPEC_PATTERN = %r{^qa/qa/specs/features/}.freeze
+
+ def initialize(mr_diff, mr_labels)
+ @mr_diff = mr_diff
+ @mr_labels = mr_labels
+ end
+
+ # Specific specs to run
+ #
+ # @return [String]
+ def qa_tests
+ return if mr_diff.empty?
+ # make paths relative to qa directory
+ return changed_files&.map { |path| path.delete_prefix("qa/") }&.join(" ") if only_spec_changes?
+ return qa_spec_directories_for_devops_stage&.join(" ") if non_qa_changes? && mr_labels.any?
+ end
+
+ # Qa framework changes
+ #
+ # @return [Boolean]
+ def framework_changes?
+ return false if mr_diff.empty?
+ return false if only_spec_changes?
+
+ changed_files
+ # TODO: expand pattern to other non spec paths that shouldn't trigger full suite
+ .select { |file_path| file_path.match?(QA_PATTERN) && !file_path.match?(SPEC_PATTERN) }
+ .any?
+ end
+
+ def quarantine_changes?
+ return false if mr_diff.empty?
+ return false if mr_diff.any? { |change| change[:new_file] || change[:deleted_file] }
+
+ files_count = 0
+ specs_count = 0
+ quarantine_specs_count = 0
+
+ mr_diff.each do |change|
+ path = change[:path]
+ next if File.directory?(File.expand_path("../#{path}", QA::Runtime::Path.qa_root))
+
+ files_count += 1
+ next unless path.match?(SPEC_PATTERN) && path.end_with?('_spec.rb')
+
+ specs_count += 1
+ quarantine_specs_count += 1 if change[:diff].match?(/^\+.*,? quarantine:/)
+ end
+
+ return false if specs_count == 0
+ return true if quarantine_specs_count == specs_count && quarantine_specs_count == files_count
+
+ false
+ end
+
+ private
+
+ # @return [Array]
+ attr_reader :mr_diff
+
+ # @return [Array]
+ attr_reader :mr_labels
+
+ # Are the changed files only qa specs?
+ #
+ # @return [Boolean] whether the changes files are only qa specs
+ def only_spec_changes?
+ changed_files.all? { |file_path| file_path =~ SPEC_PATTERN }
+ end
+
+ # Are the changed files only outside the qa directory?
+ #
+ # @return [Boolean] whether the changes files are outside of qa directory
+ def non_qa_changes?
+ changed_files.none? { |file_path| file_path =~ QA_PATTERN }
+ end
+
+ # Extract devops stage from MR labels
+ #
+ # @return [String] a devops stage
+ def devops_stage_from_mr_labels
+ mr_labels.find { |label| label =~ /^devops::/ }&.delete_prefix('devops::')
+ end
+
+ # Get qa spec directories for devops stage
+ #
+ # @return [Array] qa spec directories
+ def qa_spec_directories_for_devops_stage
+ devops_stage = devops_stage_from_mr_labels
+ return unless devops_stage
+
+ Dir.glob("qa/specs/**/*/").select { |dir| dir =~ %r{\d+_#{devops_stage}/$} }
+ end
+
+ # Change files in merge request
+ #
+ # @return [Array<String>]
+ def changed_files
+ @changed_files ||= mr_diff.map { |change| change[:path] } # rubocop:disable Rails/Pluck
+ end
+ end
+ end
+ end
+end
diff --git a/qa/qa/tools/ci/test_results.rb b/qa/qa/tools/ci/test_results.rb
new file mode 100644
index 00000000000..635b69f6ca0
--- /dev/null
+++ b/qa/qa/tools/ci/test_results.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module QA
+ module Tools
+ module Ci
+ class TestResults
+ include Helpers
+
+ def initialize(pipeline_name, test_report_job_name, report_path)
+ @pipeline_name = pipeline_name
+ @test_report_job_name = test_report_job_name
+ @report_path = report_path
+ end
+
+ # Get test report artifacts from downstream pipeline
+ #
+ # @param [String] pipeline_name
+ # @param [String] test_report_job_name
+ # @param [String] report_path
+ # @return [void]
+ def self.get(pipeline_name, test_report_job_name, report_path)
+ new(pipeline_name, test_report_job_name, report_path).download_test_results
+ end
+
+ # Download test results from child pipeline
+ #
+ # @return [void]
+ def download_test_results
+ logger.info("Fetching test results for '#{pipeline_name}'")
+
+ logger.debug(" fetching pipeline id of '#{pipeline_name}' child pipeline")
+ downstream_pipeline_id = api_get("#{pipelines_url(pipeline_id)}/bridges")
+ .find { |bridge| bridge[:name] == pipeline_name }
+ &.dig(:downstream_pipeline, :id)
+ return logger.error("Child pipeline '#{pipeline_name}' not found!") unless downstream_pipeline_id
+
+ logger.debug(" fetching job id of test report job")
+ job_id = api_get("#{pipelines_url(downstream_pipeline_id)}/jobs")
+ .find { |job| job[:name] == test_report_job_name }
+ &.fetch(:id)
+ return logger.error("Test report job '#{test_report_job_name}' not found!") unless job_id
+
+ logger.debug(" fetching test results artifact archive")
+ response = api_get("/projects/#{project_id}/jobs/#{job_id}/artifacts", raw_response: true)
+
+ logger.info("Extracting test result archive")
+ system("unzip", "-o", "-d", report_path, response.file.path)
+ end
+
+ private
+
+ attr_reader :pipeline_name, :test_report_job_name, :report_path
+
+ # Base get pipeline url
+ #
+ # @param [Integer] id
+ # @return [String]
+ def pipelines_url(id)
+ "/projects/#{project_id}/pipelines/#{id}"
+ end
+
+ # Current pipeline id
+ #
+ # @return [String]
+ def pipeline_id
+ ENV["CI_PIPELINE_ID"]
+ end
+
+ # Current project id
+ #
+ # @return [String]
+ def project_id
+ ENV["CI_PROJECT_ID"]
+ end
+ end
+ end
+ end
+end
diff --git a/qa/qa/tools/revoke_all_personal_access_tokens.rb b/qa/qa/tools/revoke_all_personal_access_tokens.rb
deleted file mode 100644
index b4fa02a36d4..00000000000
--- a/qa/qa/tools/revoke_all_personal_access_tokens.rb
+++ /dev/null
@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-require 'net/protocol'
-
-# This script revokes all personal access tokens with the name of 'api-test-token' on the host specified by GITLAB_ADDRESS
-# Required environment variables: GITLAB_USERNAME, GITLAB_PASSWORD and GITLAB_ADDRESS
-# Run `rake revoke_personal_access_tokens`
-
-module QA
- module Tools
- class RevokeAllPersonalAccessTokens
- def run
- do_run
- rescue Net::ReadTimeout
- $stdout.puts 'Net::ReadTimeout during run. Trying again'
- run
- end
-
- private
-
- def do_run
- raise ArgumentError, "Please provide GITLAB_USERNAME" unless ENV['GITLAB_USERNAME']
- raise ArgumentError, "Please provide GITLAB_PASSWORD" unless ENV['GITLAB_PASSWORD']
- raise ArgumentError, "Please provide GITLAB_ADDRESS" unless ENV['GITLAB_ADDRESS']
-
- $stdout.puts 'Running...'
-
- Runtime::Browser.visit(ENV['GITLAB_ADDRESS'], Page::Main::Login)
- Page::Main::Login.perform(&:sign_in_using_credentials)
- Page::Main::Menu.perform(&:click_edit_profile_link)
- Page::Profile::Menu.perform(&:click_access_tokens)
-
- token_name = 'api-test-token'
-
- Page::Profile::PersonalAccessTokens.perform do |tokens_page|
- while tokens_page.has_token_row_for_name?(token_name)
- tokens_page.revoke_first_token_with_name(token_name)
- print "\e[32m.\e[0m"
- end
- end
- end
- end
- end
-end
diff --git a/qa/qa/tools/revoke_user_personal_access_tokens.rb b/qa/qa/tools/revoke_user_personal_access_tokens.rb
new file mode 100644
index 00000000000..2854241f420
--- /dev/null
+++ b/qa/qa/tools/revoke_user_personal_access_tokens.rb
@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+# This script revokes all active personal access tokens owned by a given USER_ID
+# up to a given date (Date.today - 1 by default)
+# Required environment variables: USER_ID, GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS
+# Run `rake revoke_user_pats`
+
+module QA
+ module Tools
+ class RevokeUserPersonalAccessTokens
+ include Support::API
+
+ def initialize(revoke_before: (Date.today - 1).to_s, dry_run: false)
+ raise ArgumentError, "Please provide GITLAB_ADDRESS environment variable" unless ENV['GITLAB_ADDRESS']
+
+ unless ENV['GITLAB_QA_ACCESS_TOKEN']
+ raise ArgumentError, "Please provide GITLAB_QA_ACCESS_TOKEN environment variable"
+ end
+
+ raise ArgumentError, "Please provide USER_ID environment variable" unless ENV['USER_ID']
+
+ @revoke_before = Date.parse(revoke_before)
+ @dry_run = dry_run
+ @api_client = Runtime::API::Client.new(ENV['GITLAB_ADDRESS'],
+ personal_access_token: ENV['GITLAB_QA_ACCESS_TOKEN'])
+ end
+
+ def run
+ $stdout.puts 'Running...'
+
+ tokens_head_response = head Runtime::API::Request.new(@api_client,
+ "/personal_access_tokens?user_id=#{ENV['USER_ID']}",
+ per_page: "100").url
+
+ total_token_pages = tokens_head_response.headers[:x_total_pages]
+ total_tokens = tokens_head_response.headers[:x_total]
+
+ $stdout.puts "Total tokens: #{total_tokens}. Total pages: #{total_token_pages}"
+
+ tokens = fetch_tokens
+
+ revoke_tokens(tokens, @api_client, @dry_run) unless tokens.empty?
+ $stdout.puts "\nDone"
+ end
+
+ private
+
+ def fetch_tokens
+ fetched_tokens = []
+
+ page_no = 1
+
+ while page_no > 0
+ tokens_response = get Runtime::API::Request.new(@api_client,
+ "/personal_access_tokens?user_id=#{ENV['USER_ID']}",
+ page: page_no.to_s, per_page: "100").url
+
+ fetched_tokens
+ .concat(JSON.parse(tokens_response.body)
+ .select { |token| Date.parse(token["created_at"]) < @revoke_before && token['active'] }
+ .map { |token| { id: token["id"], name: token["name"], created_at: token["created_at"] } }
+ )
+
+ page_no = tokens_response.headers[:x_next_page].to_i
+ end
+
+ fetched_tokens
+ end
+
+ def revoke_tokens(tokens, api_client, dry_run = false)
+ if dry_run
+ $stdout.puts "Following #{tokens.count} tokens would be revoked:"
+ else
+ $stdout.puts "Revoking #{tokens.count} tokens..."
+ end
+
+ tokens.each do |token|
+ if dry_run
+ $stdout.puts "Token name: #{token[:name]}, id: #{token[:id]}, created at: #{token[:created_at]}"
+ else
+ request_url = Runtime::API::Request.new(api_client, "/personal_access_tokens/#{token[:id]}").url
+
+ $stdout.puts "\nRevoking token with name: #{token[:name]}, " \
+ "id: #{token[:id]}, created at: #{token[:created_at]}"
+
+ delete_response = delete(request_url)
+ dot_or_f = delete_response.code == 204 ? "\e[32m.\e[0m" : "\e[31mF - #{delete_response}\e[0m"
+ print dot_or_f
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/qa/qa/tools/test_resources_handler.rb b/qa/qa/tools/test_resources_handler.rb
index 0030a47ed55..60c6dbfc16c 100644
--- a/qa/qa/tools/test_resources_handler.rb
+++ b/qa/qa/tools/test_resources_handler.rb
@@ -27,7 +27,6 @@ module QA
include Support::API
IGNORED_RESOURCES = [
- 'QA::Resource::PersonalAccessToken',
'QA::Resource::CiVariable',
'QA::Resource::Repository::Commit',
'QA::EE::Resource::GroupIteration',
View Lorry Controller Status