From 71786ddc8e28fbd3cb3fcc4b3ff15e5962a1c82e Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Mon, 20 Feb 2023 13:49:51 +0000 Subject: Add latest changes from gitlab-org/gitlab@15-9-stable-ee --- tooling/bin/js_to_system_specs_mappings | 14 ++++ tooling/bin/view_to_js_mappings | 4 +- tooling/danger/config_files.rb | 23 +++--- tooling/danger/feature_flag.rb | 20 +++--- tooling/danger/product_intelligence.rb | 12 ++++ tooling/danger/project_helper.rb | 1 + tooling/danger/specs.rb | 84 +++++----------------- tooling/danger/stable_branch.rb | 69 ++++++++++++++++-- tooling/danger/suggestor.rb | 63 ++++++++++++++++ tooling/lib/tooling/find_codeowners.rb | 2 +- tooling/lib/tooling/helm3_client.rb | 20 +++--- tooling/lib/tooling/kubernetes_client.rb | 11 +-- tooling/lib/tooling/mappings/base.rb | 30 ++++++++ .../mappings/js_to_system_specs_mappings.rb | 61 ++++++++++++++++ .../lib/tooling/mappings/view_to_js_mappings.rb | 74 +++++++++++++++++++ tooling/lib/tooling/parallel_rspec_runner.rb | 2 + tooling/lib/tooling/test_map_packer.rb | 4 +- tooling/lib/tooling/view_to_js_mappings.rb | 77 -------------------- tooling/quality/test_level.rb | 2 +- 19 files changed, 380 insertions(+), 193 deletions(-) create mode 100755 tooling/bin/js_to_system_specs_mappings create mode 100644 tooling/danger/suggestor.rb create mode 100644 tooling/lib/tooling/mappings/base.rb create mode 100644 tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb create mode 100644 tooling/lib/tooling/mappings/view_to_js_mappings.rb delete mode 100644 tooling/lib/tooling/view_to_js_mappings.rb (limited to 'tooling') diff --git a/tooling/bin/js_to_system_specs_mappings b/tooling/bin/js_to_system_specs_mappings new file mode 100755 index 00000000000..3e9d9cb4c5f --- /dev/null +++ b/tooling/bin/js_to_system_specs_mappings @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative '../lib/tooling/mappings/js_to_system_specs_mappings' + +changes = ARGV.shift +matching_tests = ARGV.shift + +changed_files = File.read(changes).split(' ') +matching_test_files = File.read(matching_tests).split(' ') + +system_tests = Tooling::Mappings::JsToSystemSpecsMappings.new.execute(changed_files) + +File.write(matching_tests, (matching_test_files + system_tests).join(' ')) diff --git a/tooling/bin/view_to_js_mappings b/tooling/bin/view_to_js_mappings index 2cebb91892e..483003aac5e 100755 --- a/tooling/bin/view_to_js_mappings +++ b/tooling/bin/view_to_js_mappings @@ -1,10 +1,10 @@ #!/usr/bin/env ruby # frozen_string_literal: true -require_relative '../lib/tooling/view_to_js_mappings' +require_relative '../lib/tooling/mappings/view_to_js_mappings' changes = ARGV.shift output_file = ARGV.shift changed_files = File.read(changes).split(' ') -File.write(output_file, Tooling::ViewToJsMappings.new.execute(changed_files).join(' ')) +File.write(output_file, Tooling::Mappings::ViewToJsMappings.new.execute(changed_files).join(' ')) diff --git a/tooling/danger/config_files.rb b/tooling/danger/config_files.rb index 914605a3783..e165792471f 100644 --- a/tooling/danger/config_files.rb +++ b/tooling/danger/config_files.rb @@ -1,15 +1,14 @@ # frozen_string_literal: true require 'yaml' +require_relative 'suggestor' module Tooling module Danger module ConfigFiles - SUGGEST_INTRODUCED_BY_COMMENT = <<~SUGGEST_COMMENT - ```suggestion - introduced_by_url: % - ``` - SUGGEST_COMMENT + include ::Tooling::Danger::Suggestor + + MISSING_INTRODUCED_BY_REGEX = /^\+?(?\s*introduced_by_url):\s*$/.freeze CONFIG_DIRS = %w[ config/feature_flags @@ -18,14 +17,12 @@ module Tooling ].freeze def add_suggestion_for_missing_introduced_by_url - new_config_files.each do |file_name| - config_file_lines = project_helper.file_lines(file_name) - - config_file_lines.each_with_index do |added_line, i| - next unless added_line =~ /^introduced_by_url:\s?$/ - - markdown(format(SUGGEST_INTRODUCED_BY_COMMENT, url: helper.mr_web_url), file: file_name, line: i + 1) - end + new_config_files.each do |filename| + add_suggestion( + filename: filename, + regex: MISSING_INTRODUCED_BY_REGEX, + replacement: "\\k: #{helper.mr_web_url}" + ) end end diff --git a/tooling/danger/feature_flag.rb b/tooling/danger/feature_flag.rb index cef64e52af3..da0b7053af1 100644 --- a/tooling/danger/feature_flag.rb +++ b/tooling/danger/feature_flag.rb @@ -16,26 +16,22 @@ module Tooling end class Found + ATTRIBUTES = %w[name introduced_by_url rollout_issue_url milestone type group default_enabled].freeze + attr_reader :path def initialize(path) @path = path end - def raw - @raw ||= File.read(path) - end - - def group - @group ||= yaml['group'] + ATTRIBUTES.each do |attribute| + define_method(attribute) do + yaml[attribute] + end end - def default_enabled - @default_enabled ||= yaml['default_enabled'] - end - - def rollout_issue_url - @rollout_issue_url ||= yaml['rollout_issue_url'] + def raw + @raw ||= File.read(path) end def group_match_mr_label?(mr_group_label) diff --git a/tooling/danger/product_intelligence.rb b/tooling/danger/product_intelligence.rb index 58e327408a1..d25f966504f 100644 --- a/tooling/danger/product_intelligence.rb +++ b/tooling/danger/product_intelligence.rb @@ -22,6 +22,11 @@ module Tooling MSG + CHANGED_USAGE_DATA_MESSAGE = <<~MSG + Notice that implementing metrics directly in usage_data.rb has been deprecated. ([Deprecated Usage Metrics](https://docs.gitlab.com/ee/development/service_ping/usage_data.html#usage-data-metrics-guide)) + Please use [Instrumentation Classes](https://docs.gitlab.com/ee/development/service_ping/metrics_instrumentation.html) instead. + MSG + WORKFLOW_LABELS = [ APPROVED_LABEL, REVIEW_LABEL @@ -47,6 +52,13 @@ module Tooling helper.labels_to_add.concat(missing_labels) unless missing_labels.empty? end + def check_usage_data_insertions! + usage_data_changes = helper.changed_lines("lib/gitlab/usage_data.rb") + return if usage_data_changes.none? { |change| change.start_with?("+") } + + warn format(CHANGED_USAGE_DATA_MESSAGE) + end + private def convert_to_table(items) diff --git a/tooling/danger/project_helper.rb b/tooling/danger/project_helper.rb index fbf102422aa..2a77ac337a2 100644 --- a/tooling/danger/project_helper.rb +++ b/tooling/danger/project_helper.rb @@ -12,6 +12,7 @@ module Tooling sidekiq_queues specialization_labels specs + stable_branch_patch z_metadata ].freeze diff --git a/tooling/danger/specs.rb b/tooling/danger/specs.rb index 6c0459a4344..f95a798d53e 100644 --- a/tooling/danger/specs.rb +++ b/tooling/danger/specs.rb @@ -1,12 +1,14 @@ # frozen_string_literal: true +require_relative 'suggestor' + module Tooling module Danger module Specs + include ::Tooling::Danger::Suggestor + SPEC_FILES_REGEX = 'spec/' EE_PREFIX = 'ee/' - MATCH_WITH_ARRAY_REGEX = /(?to\(?\s*)(?match|eq)(?[( ]?\[[^\]]+)/.freeze - MATCH_WITH_ARRAY_REPLACEMENT = '\kmatch_array\k' PROJECT_FACTORIES = %w[ :project @@ -29,22 +31,18 @@ module Tooling /x.freeze PROJECT_FACTORY_REPLACEMENT = '\klet_it_be\k' - SUGGESTION_MARKDOWN = <<~SUGGESTION_MARKDOWN - ```suggestion - %s - ``` - SUGGESTION_MARKDOWN - - MATCH_WITH_ARRAY_SUGGESTION = <<~SUGGEST_COMMENT - If order of the result is not important, please consider using `match_array` to avoid flakiness. - SUGGEST_COMMENT - PROJECT_FACTORY_SUGGESTION = <<~SUGGEST_COMMENT Project creations are very slow. Use `let_it_be`, `build` or `build_stubbed` if possible. See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#optimize-factory-usage) for background information and alternative options. SUGGEST_COMMENT + MATCH_WITH_ARRAY_REGEX = /(?to\(?\s*)(?match|eq)(?[( ]?\[(?=.*,)[^\]]+)/.freeze + MATCH_WITH_ARRAY_REPLACEMENT = '\kmatch_array\k' + MATCH_WITH_ARRAY_SUGGESTION = <<~SUGGEST_COMMENT + If order of the result is not important, please consider using `match_array` to avoid flakiness. + SUGGEST_COMMENT + RSPEC_TOP_LEVEL_DESCRIBE_REGEX = /^\+.?RSpec\.describe(.+)/.freeze FEATURE_CATEGORY_SUGGESTION = <<~SUGGESTION_MARKDOWN Consider adding `feature_category: ` for this example if it is not set already. @@ -69,19 +67,19 @@ module Tooling def add_suggestions_for_match_with_array(filename) add_suggestion( - filename, - MATCH_WITH_ARRAY_REGEX, - MATCH_WITH_ARRAY_SUGGESTION, - MATCH_WITH_ARRAY_REPLACEMENT + filename: filename, + regex: MATCH_WITH_ARRAY_REGEX, + replacement: MATCH_WITH_ARRAY_REPLACEMENT, + comment_text: MATCH_WITH_ARRAY_SUGGESTION ) end def add_suggestions_for_project_factory_usage(filename) add_suggestion( - filename, - PROJECT_FACTORY_REGEX, - PROJECT_FACTORY_SUGGESTION, - PROJECT_FACTORY_REPLACEMENT + filename: filename, + regex: PROJECT_FACTORY_REGEX, + replacement: PROJECT_FACTORY_REPLACEMENT, + comment_text: PROJECT_FACTORY_SUGGESTION ) end @@ -103,53 +101,9 @@ module Tooling suggested_line = file_lines[line_number] - text = format(comment(FEATURE_CATEGORY_SUGGESTION), suggested_line: suggested_line) - markdown(text, file: filename, line: line_number + 1) + markdown(comment(FEATURE_CATEGORY_SUGGESTION, suggested_line), file: filename, line: line_number.succ) end end - - private - - def added_lines_matching(filename, regex) - helper.changed_lines(filename).grep(/\A\+( )?/).grep(regex) - end - - def add_suggestion(filename, regex, comment_text, replacement = nil, exclude = nil) - added_lines = added_lines_matching(filename, regex) - - return if added_lines.empty? - - spec_file_lines = project_helper.file_lines(filename) - - added_lines.each_with_object([]) do |added_line, processed_line_numbers| - line_number = find_line_number(spec_file_lines, added_line.delete_prefix('+'), exclude_indexes: processed_line_numbers) - next unless line_number - next if !exclude.nil? && added_line.include?(exclude) - - processed_line_numbers << line_number - - suggested_line = spec_file_lines[line_number] - suggested_line = suggested_line.gsub(regex, replacement) unless replacement.nil? - - text = format(comment(comment_text), suggested_line: suggested_line) - markdown(text, file: filename, line: line_number.succ) - end - end - - def comment(comment_text) - <<~COMMENT_BODY.chomp - #{SUGGESTION_MARKDOWN} - #{comment_text} - COMMENT_BODY - end - - def find_line_number(file_lines, searched_line, exclude_indexes: []) - _, index = file_lines.each_with_index.find do |file_line, index| - file_line == searched_line && !exclude_indexes.include?(index) - end - - index - end end end end diff --git a/tooling/danger/stable_branch.rb b/tooling/danger/stable_branch.rb index 6c0b94b4f06..8fac1cc5fbf 100644 --- a/tooling/danger/stable_branch.rb +++ b/tooling/danger/stable_branch.rb @@ -6,6 +6,7 @@ module Tooling VersionApiError = Class.new(StandardError) STABLE_BRANCH_REGEX = %r{\A(?\d+-\d+)-stable-ee\z}.freeze + FAILING_PACKAGE_AND_TEST_STATUSES = %w[manual canceled].freeze # rubocop:disable Lint/MixedRegexpCaptureTypes VERSION_REGEX = %r{ @@ -32,26 +33,69 @@ module Tooling This branch is meant for backporting bug fixes. If this MR qualifies please add the `type::bug` label. #{MAINTENANCE_POLICY_MESSAGE} MSG - VERSION_ERROR_MESSAGE = <<~MSG - Patches are only being accepted on the most recent 3 minor versions of GitLab. #{MAINTENANCE_POLICY_MESSAGE} + VERSION_WARNING_MESSAGE = <<~MSG + Backporting to older releases requires an [exception request process](https://docs.gitlab.com/ee/policy/maintenance.html#backporting-to-older-releases) MSG FAILED_VERSION_REQUEST_MESSAGE = <<~MSG There was a problem checking if this is a qualified version for backporting. Re-running this job may fix the problem. MSG + PIPELINE_EXPEDITE_ERROR_MESSAGE = <<~MSG + ~"pipeline:expedite" is not allowed on stable branches because it causes the `e2e:package-and-test` job to be skipped. + MSG + + NEEDS_PACKAGE_AND_TEST_MESSAGE = <<~MSG + The `e2e:package-and-test` job is not present or needs to be triggered manually. Please start the `e2e:package-and-test` + job and re-run `danger-review`. + MSG + + WARN_PACKAGE_AND_TEST_MESSAGE = <<~MSG + The `e2e:package-and-test` job needs to succeed or have approval from a Software Engineer in Test. See the section below + for more details. + MSG + # rubocop:disable Style/SignalException def check! - return unless stable_target_branch && !helper.security_mr? + return unless non_security_stable_branch? fail FEATURE_ERROR_MESSAGE if has_feature_label? fail BUG_ERROR_MESSAGE unless has_bug_label? - fail VERSION_ERROR_MESSAGE unless targeting_patchable_version? + + warn VERSION_WARNING_MESSAGE unless targeting_patchable_version? + + return if has_flaky_failure_label? || has_only_documentation_changes? + + fail PIPELINE_EXPEDITE_ERROR_MESSAGE if has_pipeline_expedite_label? + + status = package_and_test_status + + if status.nil? || FAILING_PACKAGE_AND_TEST_STATUSES.include?(status) # rubocop:disable Style/GuardClause + fail NEEDS_PACKAGE_AND_TEST_MESSAGE + else + warn WARN_PACKAGE_AND_TEST_MESSAGE unless status == 'success' + end end # rubocop:enable Style/SignalException + def non_security_stable_branch? + !!stable_target_branch && !helper.security_mr? + end + private + def package_and_test_status + mr_head_pipeline_id = gitlab.mr_json.dig('head_pipeline', 'id') + return unless mr_head_pipeline_id + + pipeline_bridges = gitlab.api.pipeline_bridges(helper.mr_target_project_id, mr_head_pipeline_id) + package_and_test_pipeline = pipeline_bridges&.find { |j| j['name'] == 'e2e:package-and-test' } + + return unless package_and_test_pipeline + + package_and_test_pipeline['status'] + end + def stable_target_branch helper.mr_target_branch.match(STABLE_BRANCH_REGEX) end @@ -64,12 +108,27 @@ module Tooling helper.mr_has_labels?('type::bug') end + def has_pipeline_expedite_label? + helper.mr_has_labels?('pipeline:expedite') + end + + def has_flaky_failure_label? + helper.mr_has_labels?('failure::flaky-test') + end + + def has_only_documentation_changes? + categories_changed = helper.changes_by_category.keys + return false unless categories_changed.size == 1 + return true if categories_changed.first == :docs + + false + end + def targeting_patchable_version? raise VersionApiError if last_three_minor_versions.empty? last_three_minor_versions.include?(targeted_version) rescue VersionApiError - # don't fail the job since we do not know the recent versions warn FAILED_VERSION_REQUEST_MESSAGE true end diff --git a/tooling/danger/suggestor.rb b/tooling/danger/suggestor.rb new file mode 100644 index 00000000000..ffda98e67d0 --- /dev/null +++ b/tooling/danger/suggestor.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true +module Tooling + module Danger + module Suggestor + # For file lines matching `regex` adds suggestion `replacement` with `comment_text` added. + def add_suggestion(filename:, regex:, replacement: nil, comment_text: nil, exclude: nil) + added_lines = added_lines_matching(filename, regex) + + return if added_lines.empty? + + file_lines = project_helper.file_lines(filename) + + added_lines.each_with_object([]) do |added_line, processed_line_numbers| + line_number = find_line_number(file_lines, added_line.delete_prefix('+'), +exclude_indexes: processed_line_numbers) + + next unless line_number + next if !exclude.nil? && added_line.include?(exclude) + + processed_line_numbers << line_number + + if replacement + suggestion_text = file_lines[line_number] + suggestion_text = suggestion_text.gsub(regex, replacement) + end + + markdown(comment(comment_text, suggestion_text), file: filename, line: line_number.succ) + end + end + + private + + def added_lines_matching(filename, regex) + helper.changed_lines(filename).grep(/\A\+( )?/).grep(regex) + end + + def find_line_number(file_lines, searched_line, exclude_indexes: []) + _, index = file_lines.each_with_index.find do |file_line, index| + file_line == searched_line && !exclude_indexes.include?(index) # rubocop:disable Rails/NegateInclude + end + + index + end + + def comment(comment_text = nil, suggested_line = nil) + if suggested_line + suggestion_text = <<~SUGGESTION + ```suggestion + %s + ``` + SUGGESTION + end + + comment_body = <<~COMMENT_BODY.chomp + #{suggestion_text} + #{comment_text} + COMMENT_BODY + + format(comment_body.chomp, suggested_line: suggested_line) + end + end + end +end diff --git a/tooling/lib/tooling/find_codeowners.rb b/tooling/lib/tooling/find_codeowners.rb index 6a90f86eecc..cc37d4db1ec 100644 --- a/tooling/lib/tooling/find_codeowners.rb +++ b/tooling/lib/tooling/find_codeowners.rb @@ -89,7 +89,7 @@ module Tooling end def consolidate_paths(matched_files) - matched_files.group_by(&File.method(:dirname)).flat_map do |dir, files| + matched_files.group_by { |file| File.dirname(file) }.flat_map do |dir, files| # First line is the dir itself if find_dir_maxdepth_1(dir).lines.drop(1).sort == files.sort "#{dir}\n" diff --git a/tooling/lib/tooling/helm3_client.rb b/tooling/lib/tooling/helm3_client.rb index d4e7faa802e..9059387351a 100644 --- a/tooling/lib/tooling/helm3_client.rb +++ b/tooling/lib/tooling/helm3_client.rb @@ -8,34 +8,32 @@ module Tooling class Helm3Client CommandFailedError = Class.new(StandardError) - attr_reader :namespace - RELEASE_JSON_ATTRIBUTES = %w[name revision updated status chart app_version namespace].freeze PAGINATION_SIZE = 256 # Default helm list pagination size - Release = Struct.new(:name, :revision, :last_update, :status, :chart, :app_version, :namespace) do + Release = Struct.new(:name, :namespace, :revision, :updated, :status, :chart, :app_version, keyword_init: true) do def revision @revision ||= self[:revision].to_i end def last_update - @last_update ||= self[:last_update] ? Time.parse(self[:last_update]) : nil + @last_update ||= self[:updated] ? Time.parse(self[:updated]) : nil end end # A single page of data and the corresponding page number. Page = Struct.new(:releases, :number) - def initialize(namespace:) - @namespace = namespace - end - def releases(args: []) each_release(args) end - def delete(release_name:) - run_command(['uninstall', release_name]) + def delete(release_name:, namespace: nil) + release_name = Array(release_name) + + release_name.each do |release| + run_command(['uninstall', '--namespace', (namespace || release), release]) + end end private @@ -66,7 +64,7 @@ module Tooling releases = JSON.parse(response) # rubocop:disable Gitlab/Json releases.map do |release| - Release.new(*release.values_at(*RELEASE_JSON_ATTRIBUTES)) + Release.new(release.slice(*RELEASE_JSON_ATTRIBUTES)) end rescue ::JSON::ParserError => ex puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" # rubocop:disable Rails/Output diff --git a/tooling/lib/tooling/kubernetes_client.rb b/tooling/lib/tooling/kubernetes_client.rb index 1d7b924e2c3..ab914db5777 100644 --- a/tooling/lib/tooling/kubernetes_client.rb +++ b/tooling/lib/tooling/kubernetes_client.rb @@ -6,8 +6,9 @@ require_relative '../../../lib/gitlab/popen' unless defined?(Gitlab::Popen) module Tooling class KubernetesClient - RESOURCE_LIST = 'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd' - CommandFailedError = Class.new(StandardError) + RESOURCE_LIST = 'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd' + K8S_ALLOWED_NAMESPACES_REGEX = /^review-(?!apps).+/.freeze + CommandFailedError = Class.new(StandardError) attr_reader :namespace @@ -129,14 +130,16 @@ module Tooling command = [ 'get', 'namespace', - "-l tls=review-apps-tls", # Get only namespaces used for review-apps "--sort-by='{.metadata.creationTimestamp}'", '-o json' ] response = run_command(command) - resources_created_before_date(response, created_before) + stale_namespaces = resources_created_before_date(response, created_before) + + # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby. + stale_namespaces.select { |ns| K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) } end def resources_created_before_date(response, date) diff --git a/tooling/lib/tooling/mappings/base.rb b/tooling/lib/tooling/mappings/base.rb new file mode 100644 index 00000000000..93d3a967114 --- /dev/null +++ b/tooling/lib/tooling/mappings/base.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require_relative '../../../../lib/gitlab_edition' + +# Returns system specs files that are related to the JS files that were changed in the MR. +module Tooling + module Mappings + class Base + # Input: A list of space-separated files + # Output: A list of space-separated specs files (JS, Ruby, ...) + def execute(changed_files) + raise "Not Implemented" + end + + # Input: A list of space-separated files + # Output: array/hash of files + def filter_files(changed_files) + raise "Not Implemented" + end + + # Input: A folder + # Output: An array of folders, each prefixed with a GitLab edition + def folders_for_available_editions(base_folder) + foss_prefix = base_folder + extension_prefixes = ::GitlabEdition.extensions.map { |prefix| "#{prefix}/#{foss_prefix}" } + [foss_prefix, *extension_prefixes] + end + end + end +end diff --git a/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb b/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb new file mode 100644 index 00000000000..365e466011b --- /dev/null +++ b/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'active_support/inflector' + +require_relative 'base' +require_relative '../../../../lib/gitlab_edition' + +# Returns system specs files that are related to the JS files that were changed in the MR. +module Tooling + module Mappings + class JsToSystemSpecsMappings < Base + def initialize(js_base_folder: 'app/assets/javascripts', system_specs_base_folder: 'spec/features') + @js_base_folder = js_base_folder + @js_base_folders = folders_for_available_editions(js_base_folder) + @system_specs_base_folder = system_specs_base_folder + + # Cannot be extracted to a constant, as it depends on a variable + @first_js_folder_extract_regexp = %r{ + (?:.*/)? # Skips the GitLab edition (e.g. ee/, jh/) + #{@js_base_folder}/ # Most likely app/assets/javascripts/ + ([\w-]*) # Captures the first folder + }x + end + + def execute(changed_files) + filter_files(changed_files).flat_map do |edition, js_files| + js_keywords_regexp = Regexp.union(construct_js_keywords(js_files)) + + system_specs_for_edition(edition).select do |system_spec_file| + system_spec_file if js_keywords_regexp.match?(system_spec_file) + end + end + end + + # Keep the files that are in the @js_base_folders folders + # + # Returns a hash, where the key is the GitLab edition, and the values the JS specs + def filter_files(changed_files) + selected_files = changed_files.select do |filename| + filename.start_with?(*@js_base_folders) && File.exist?(filename) + end + + selected_files.group_by { |filename| filename[/^#{Regexp.union(::GitlabEdition.extensions)}/] } + end + + # Extract keywords in the JS filenames to be used for searching matching system specs + def construct_js_keywords(js_files) + js_files.map do |js_file| + filename = js_file.scan(@first_js_folder_extract_regexp).flatten.first + filename.singularize + end.uniq + end + + def system_specs_for_edition(edition) + all_files_in_folders_glob = File.join(@system_specs_base_folder, '**', '*') + all_files_in_folders_glob = File.join(edition, all_files_in_folders_glob) if edition + Dir[all_files_in_folders_glob].select { |f| File.file?(f) } + end + end + end +end diff --git a/tooling/lib/tooling/mappings/view_to_js_mappings.rb b/tooling/lib/tooling/mappings/view_to_js_mappings.rb new file mode 100644 index 00000000000..db80eb9bfe8 --- /dev/null +++ b/tooling/lib/tooling/mappings/view_to_js_mappings.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require_relative 'base' +require_relative '../../../../lib/gitlab_edition' + +# Returns JS files that are related to the Rails views files that were changed in the MR. +module Tooling + module Mappings + class ViewToJsMappings < Base + # The HTML attribute value pattern we're looking for to match an HTML file to a JS file. + HTML_ATTRIBUTE_VALUE_REGEXP = /js-[-\w]+/.freeze + + # Search for Rails partials included in an HTML file + RAILS_PARTIAL_INVOCATION_REGEXP = %r{(?:render|render_if_exist)(?: |\()(?:partial: ?)?['"]([\w/-]+)['"]}.freeze + + def initialize(view_base_folder: 'app/views', js_base_folder: 'app/assets/javascripts') + @view_base_folders = folders_for_available_editions(view_base_folder) + @js_base_folders = folders_for_available_editions(js_base_folder) + end + + def execute(changed_files) + changed_view_files = filter_files(changed_files) + + partials = changed_view_files.flat_map do |file| + find_partials(file) + end + + files_to_scan = changed_view_files + partials + js_tags = files_to_scan.flat_map do |file| + find_pattern_in_file(file, HTML_ATTRIBUTE_VALUE_REGEXP) + end + js_tags_regexp = Regexp.union(js_tags) + + @js_base_folders.flat_map do |js_base_folder| + Dir["#{js_base_folder}/**/*.{js,vue}"].select do |js_file| + file_content = File.read(js_file) + js_tags_regexp.match?(file_content) + end + end + end + + # Keep the files that are in the @view_base_folders folder + def filter_files(changed_files) + changed_files.select do |filename| + filename.start_with?(*@view_base_folders) && + File.exist?(filename) + end + end + + # Note: We only search for partials with depth 1. We don't do recursive search, as + # it is probably not necessary for a first iteration. + def find_partials(file) + partial_paths = find_pattern_in_file(file, RAILS_PARTIAL_INVOCATION_REGEXP) + partial_paths.flat_map do |partial_path| + view_file_folder = File.dirname(file) + partial_relative_folder = File.dirname(partial_path) + + dirname = + if partial_relative_folder == '.' # The partial is in the same folder as the HTML file + view_file_folder + else + File.join(view_file_folder, partial_relative_folder) + end + + Dir["#{dirname}/_#{File.basename(partial_path)}.*"] + end + end + + def find_pattern_in_file(file, pattern) + File.read(file).scan(pattern).flatten.uniq + end + end + end +end diff --git a/tooling/lib/tooling/parallel_rspec_runner.rb b/tooling/lib/tooling/parallel_rspec_runner.rb index b482160d3c0..b1ddc91e831 100644 --- a/tooling/lib/tooling/parallel_rspec_runner.rb +++ b/tooling/lib/tooling/parallel_rspec_runner.rb @@ -43,6 +43,8 @@ module Tooling return end + Knapsack.logger.info "Running command: #{rspec_command.join(' ')}" + exec(*rspec_command) end diff --git a/tooling/lib/tooling/test_map_packer.rb b/tooling/lib/tooling/test_map_packer.rb index 151ce88111f..15191e35c54 100644 --- a/tooling/lib/tooling/test_map_packer.rb +++ b/tooling/lib/tooling/test_map_packer.rb @@ -6,11 +6,11 @@ module Tooling MARKER = 1 def pack(map) - map.transform_values(&method(:create_tree_from_tests)) + map.transform_values { |tests| create_tree_from_tests(tests) } end def unpack(compact_map) - compact_map.transform_values(&method(:retrieve_tests_from_tree)) + compact_map.transform_values { |tree| retrieve_tests_from_tree(tree) } end private diff --git a/tooling/lib/tooling/view_to_js_mappings.rb b/tooling/lib/tooling/view_to_js_mappings.rb deleted file mode 100644 index 76704a04469..00000000000 --- a/tooling/lib/tooling/view_to_js_mappings.rb +++ /dev/null @@ -1,77 +0,0 @@ -# frozen_string_literal: true - -require_relative '../../../lib/gitlab_edition' - -# Returns JS files that are related to the Rails views files that were changed in the MR. -module Tooling - class ViewToJsMappings - # The HTML attribute value pattern we're looking for to match an HTML file to a JS file. - HTML_ATTRIBUTE_VALUE_REGEXP = /js-[-\w]+/.freeze - - # Search for Rails partials included in an HTML file - RAILS_PARTIAL_INVOCATION_REGEXP = %r{(?:render|render_if_exist)(?: |\()(?:partial: ?)?['"]([\w/-]+)['"]}.freeze - - def initialize(view_base_folder: 'app/views', js_base_folder: 'app/assets/javascripts') - @view_base_folders = folders_for_available_editions(view_base_folder) - @js_base_folders = folders_for_available_editions(js_base_folder) - end - - def execute(changed_files) - changed_view_files = view_files(changed_files) - - partials = changed_view_files.flat_map do |file| - find_partials(file) - end - - files_to_scan = changed_view_files + partials - js_tags = files_to_scan.flat_map do |file| - find_pattern_in_file(file, HTML_ATTRIBUTE_VALUE_REGEXP) - end - js_tags_regexp = Regexp.union(js_tags) - - @js_base_folders.flat_map do |js_base_folder| - Dir["#{js_base_folder}/**/*.{js,vue}"].select do |js_file| - file_content = File.read(js_file) - js_tags_regexp.match?(file_content) - end - end - end - - # Keep the files that are in the @view_base_folders folder - def view_files(changed_files) - changed_files.select do |filename| - filename.start_with?(*@view_base_folders) && - File.exist?(filename) - end - end - - def folders_for_available_editions(base_folder) - foss_prefix = base_folder - extension_prefixes = ::GitlabEdition.extensions.map { |prefix| "#{prefix}/#{foss_prefix}" } - [foss_prefix, *extension_prefixes] - end - - # Note: We only search for partials with depth 1. We don't do recursive search, as - # it is probably not necessary for a first iteration. - def find_partials(file) - partial_paths = find_pattern_in_file(file, RAILS_PARTIAL_INVOCATION_REGEXP) - partial_paths.flat_map do |partial_path| - view_file_folder = File.dirname(file) - partial_relative_folder = File.dirname(partial_path) - - dirname = - if partial_relative_folder == '.' # The partial is in the same folder as the HTML file - view_file_folder - else - File.join(view_file_folder, partial_relative_folder) - end - - Dir["#{dirname}/_#{File.basename(partial_path)}.*"] - end - end - - def find_pattern_in_file(file, pattern) - File.read(file).scan(pattern).flatten.uniq - end - end -end diff --git a/tooling/quality/test_level.rb b/tooling/quality/test_level.rb index 31196290de4..eeda135f3ee 100644 --- a/tooling/quality/test_level.rb +++ b/tooling/quality/test_level.rb @@ -115,7 +115,7 @@ module Quality def prefixes_for_regex return '' if prefixes.empty? - regex_prefix = prefixes.map(&Regexp.method(:escape)).join('|') + regex_prefix = prefixes.map { |prefix| Regexp.escape(prefix) }.join('|') "(#{regex_prefix})" end -- cgit v1.2.1