diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-11-18 13:16:36 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-11-18 13:16:36 +0000 |
commit | 311b0269b4eb9839fa63f80c8d7a58f32b8138a0 (patch) | |
tree | 07e7870bca8aed6d61fdcc810731c50d2c40af47 /tooling | |
parent | 27909cef6c4170ed9205afa7426b8d3de47cbb0c (diff) | |
download | gitlab-ce-311b0269b4eb9839fa63f80c8d7a58f32b8138a0.tar.gz |
Add latest changes from gitlab-org/gitlab@14-5-stable-eev14.5.0-rc42
Diffstat (limited to 'tooling')
-rwxr-xr-x | tooling/bin/find_change_diffs | 38 | ||||
-rwxr-xr-x | tooling/bin/find_changes | 2 | ||||
-rwxr-xr-x | tooling/bin/qa/package_and_qa_check | 45 | ||||
-rw-r--r-- | tooling/danger/changelog.rb | 2 | ||||
-rw-r--r-- | tooling/danger/product_intelligence.rb | 57 | ||||
-rw-r--r-- | tooling/danger/project_helper.rb | 41 | ||||
-rw-r--r-- | tooling/deprecations/docs.rb | 6 | ||||
-rw-r--r-- | tooling/lib/tooling/image.rb (renamed from tooling/lib/tooling/images.rb) | 0 | ||||
-rw-r--r-- | tooling/quality/test_level.rb | 2 | ||||
-rw-r--r-- | tooling/rspec_flaky/report.rb | 2 |
10 files changed, 134 insertions, 61 deletions
diff --git a/tooling/bin/find_change_diffs b/tooling/bin/find_change_diffs new file mode 100755 index 00000000000..7857945ea74 --- /dev/null +++ b/tooling/bin/find_change_diffs @@ -0,0 +1,38 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'gitlab' +require 'pathname' + +# This script saves the diffs of changes in an MR to the directory specified as the first argument +# +# It exits with a success code if diffs are found and saved, or if there are no changes, including if the script runs in +# a pipeline that is not for a merge request. + +gitlab_token = ENV.fetch('PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE') +gitlab_endpoint = ENV.fetch('CI_API_V4_URL') +mr_project_path = ENV['CI_MERGE_REQUEST_PROJECT_PATH'] +mr_iid = ENV['CI_MERGE_REQUEST_IID'] + +puts "CI_MERGE_REQUEST_PROJECT_PATH is missing." if mr_project_path.to_s.empty? +puts "CI_MERGE_REQUEST_IID is missing." if mr_iid.to_s.empty? + +unless mr_project_path && mr_iid + puts "Exiting as this does not appear to be a merge request pipeline." + exit +end + +abort("ERROR: Please specify a directory to write MR diffs into.") if ARGV.empty? +output_diffs_dir = Pathname.new(ARGV.shift).expand_path + +Gitlab.configure do |config| + config.endpoint = gitlab_endpoint + config.private_token = gitlab_token +end + +Gitlab.merge_request_changes(mr_project_path, mr_iid).changes.each do |change| + next if change['diff'].empty? + + output_diffs_dir.join(File.dirname(change['new_path'])).mkpath + output_diffs_dir.join("#{change['new_path']}.diff").write(change['diff']) +end diff --git a/tooling/bin/find_changes b/tooling/bin/find_changes index 466510ccb19..20df085879a 100755 --- a/tooling/bin/find_changes +++ b/tooling/bin/find_changes @@ -3,7 +3,7 @@ require 'gitlab' -gitlab_token = ENV.fetch('DANGER_GITLAB_API_TOKEN', '') +gitlab_token = ENV.fetch('PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE', '') gitlab_endpoint = ENV.fetch('CI_API_V4_URL') mr_project_path = ENV.fetch('CI_MERGE_REQUEST_PROJECT_PATH') mr_iid = ENV.fetch('CI_MERGE_REQUEST_IID') diff --git a/tooling/bin/qa/package_and_qa_check b/tooling/bin/qa/package_and_qa_check new file mode 100755 index 00000000000..21deb0fcd2d --- /dev/null +++ b/tooling/bin/qa/package_and_qa_check @@ -0,0 +1,45 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'pathname' + +# This script checks if the package-and-qa job should trigger downstream pipelines to run the QA suite. +# +# It assumes the first argument is a directory of files containing diffs of changes from an MR +# (e.g., created by tooling/bin/find_change_diffs). It exits with a success code if there are no diffs, or if the diffs +# are suitable to run QA tests. +# +# The script will abort (exit code 1) if the argument is missing. +# +# The following condition will result in a failure code (2), indicating that package-and-qa should not run: +# +# - If the changes only include tests being put in quarantine + +abort("ERROR: Please specify the directory containing MR diffs.") if ARGV.empty? +diffs_dir = Pathname.new(ARGV.shift).expand_path + +# Run package-and-qa if there are no diffs. E.g., in scheduled pipelines +exit 0 if diffs_dir.glob('**/*').empty? + +files_count = 0 +specs_count = 0 +quarantine_specs_count = 0 + +diffs_dir.glob('**/*').each do |path| + next if path.directory? + + files_count += 1 + next unless path.to_s.end_with?('_spec.rb.diff') + + specs_count += 1 + quarantine_specs_count += 1 if path.read.match?(/^\+.*, quarantine:/) +end + +# Run package-and-qa if there are no specs. E.g., when the MR changes QA framework files. +exit 0 if specs_count == 0 + +# Skip package-and-qa if there are only specs being put in quarantine. +exit 2 if quarantine_specs_count == specs_count && quarantine_specs_count == files_count + +# Run package-and-qa under any other circumstances. E.g., if there are specs being put in quarantine but there are also +# other changes that might need to be tested. diff --git a/tooling/danger/changelog.rb b/tooling/danger/changelog.rb index c053d366199..fbf8ae931e2 100644 --- a/tooling/danger/changelog.rb +++ b/tooling/danger/changelog.rb @@ -6,7 +6,7 @@ module Tooling module Danger module Changelog NO_CHANGELOG_LABELS = [ - 'tooling', + 'type::tooling', 'tooling::pipelines', 'tooling::workflow', 'ci-build', diff --git a/tooling/danger/product_intelligence.rb b/tooling/danger/product_intelligence.rb index 848f99eeff5..72fc8deac43 100644 --- a/tooling/danger/product_intelligence.rb +++ b/tooling/danger/product_intelligence.rb @@ -9,71 +9,20 @@ module Tooling 'product intelligence::review pending' ].freeze - TRACKING_FILES = [ - 'lib/gitlab/tracking.rb', - 'spec/lib/gitlab/tracking_spec.rb', - 'app/helpers/tracking_helper.rb', - 'spec/helpers/tracking_helper_spec.rb', - 'app/assets/javascripts/tracking/index.js', - 'app/assets/javascripts/tracking/constants.js', - 'app/assets/javascripts/tracking/get_standard_context.js', - 'spec/frontend/tracking/get_standard_context_spec.js', - 'spec/frontend/tracking_spec.js', - 'generator_templates/usage_metric_definition/metric_definition.yml', - 'lib/generators/gitlab/usage_metric/usage_metric_generator.rb', - 'lib/generators/gitlab/usage_metric_definition_generator.rb', - 'lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb', - 'spec/lib/generators/gitlab/usage_metric_generator_spec.rb', - 'spec/lib/generators/gitlab/usage_metric_definition_generator_spec.rb', - 'spec/lib/generators/gitlab/usage_metric_definition/redis_hll_generator_spec.rb', - 'config/metrics/schema.json' - ].freeze - def missing_labels return [] if !helper.ci? || helper.mr_has_labels?('growth experiment') labels = [] labels << 'product intelligence' unless helper.mr_has_labels?('product intelligence') - labels << 'product intelligence::review pending' unless helper.mr_has_labels?(WORKFLOW_LABELS) + labels << 'product intelligence::review pending' unless has_workflow_labels? labels end - def matching_changed_files - tracking_changed_files = all_changed_files & TRACKING_FILES - usage_data_changed_files = all_changed_files.grep(%r{(usage_data)}) - - usage_data_changed_files + tracking_changed_files + metrics_changed_files + snowplow_changed_files - end - private - def all_changed_files - helper.all_changed_files - end - - def metrics_changed_files - all_changed_files.grep(%r{((ee/)?config/metrics/.*\.yml)}) - end - - def matching_files?(file, extension:, pattern:) - return unless file.end_with?(extension) - - helper.changed_lines(file).grep(pattern).any? - end - - def snowplow_changed_files - js_patterns = Regexp.union( - 'Tracking.event', - /\btrack\(/, - 'data-track-action' - ) - all_changed_files.select do |file| - matching_files?(file, extension: '.rb', pattern: %r{Gitlab::Tracking\.(event|enabled\?|options)$}) || - matching_files?(file, extension: '.js', pattern: js_patterns) || - matching_files?(file, extension: '.vue', pattern: js_patterns) || - matching_files?(file, extension: '.haml', pattern: %r{data: \{ track}) - end + def has_workflow_labels? + (WORKFLOW_LABELS & helper.mr_labels).any? end end end diff --git a/tooling/danger/project_helper.rb b/tooling/danger/project_helper.rb index c552a75bba8..5d338393f90 100644 --- a/tooling/danger/project_helper.rb +++ b/tooling/danger/project_helper.rb @@ -22,12 +22,12 @@ module Tooling ce_ee_vue_templates ci_templates datateam - metadata feature_flag roulette sidekiq_queues specialization_labels specs + z_metadata ].freeze MESSAGE_PREFIX = '==>' @@ -38,10 +38,34 @@ module Tooling %r{\A((ee|jh)/)?config/feature_flags/} => :feature_flag, + %r{doc/api/usage_data.md} => [:product_intelligence], + %r{\Adoc/.*(\.(md|png|gif|jpg|yml))\z} => :docs, %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs, %r{\Adata/whats_new/} => :docs, + %r{\A((ee|jh)/)?app/finders/(.+/)?integrations/} => [:integrations_be, :database, :backend], + [%r{\A((ee|jh)/)?db/(geo/)?(migrate|post_migrate)/}, %r{(:integrations|:\w+_tracker_data)\b}] => [:integrations_be, :database, :migration], + [%r{\A((ee|jh)/)?(app|lib)/.+\.rb}, %r{\b(Integrations::|\.execute_(integrations|hooks))\b}] => [:integrations_be, :backend], + %r{\A( + ((ee|jh)/)?app/((?!.*clusters)(?!.*alert_management)(?!.*views)(?!.*assets).+/)?integration.+ | + ((ee|jh)/)?app/((?!.*search).+/)?project_service.+ | + ((ee|jh)/)?app/(models|helpers|workers|services|controllers)/(.+/)?(jira_connect.+|.*hook.+) | + ((ee|jh)/)?app/controllers/(.+/)?oauth/jira/.+ | + ((ee|jh)/)?app/services/(.+/)?jira.+ | + ((ee|jh)/)?app/workers/(.+/)?(propagate_integration.+|irker_worker\.rb) | + ((ee|jh)/)?lib/(.+/)?(atlassian|data_builder|hook_data)/.+ | + ((ee|jh)/)?lib/(.+/)?.*integration.+ | + ((ee|jh)/)?lib/(.+/)?api/v3/github\.rb | + ((ee|jh)/)?lib/(.+/)?api/github/entities\.rb + )\z}x => [:integrations_be, :backend], + + %r{\A( + ((ee|jh)/)?app/(views|assets)/((?!.*clusters)(?!.*alerts_settings).+/)?integration.+ | + ((ee|jh)/)?app/(views|assets)/(.+/)?jira_connect.+ | + ((ee|jh)/)?app/(views|assets)/((?!.*filtered_search).+/)?hooks?.+ + )\z}x => [:integrations_fe, :frontend], + %r{\A( app/assets/javascripts/tracking/.*\.js | spec/frontend/tracking/.*\.js | @@ -82,6 +106,7 @@ module Tooling %r{\A((ee|jh)/)?app/finders/} => [:database, :backend], %r{\Arubocop/cop/migration(/|\.rb)} => :database, + %r{\A(\.ruby-version\z|\.nvmrc\z|\.tool-versions\z)} => :tooling, %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :tooling, %r{\A\.codeclimate\.yml\z} => :tooling, %r{\Alefthook.yml\z} => :tooling, @@ -100,6 +125,7 @@ module Tooling %r{\A((ee|jh)/)?spec/support/shared_contexts/features/} => :test, %r{\A((ee|jh)/)?spec/support/helpers/features/} => :test, + %r{\A((spec/)?lib/generators/gitlab/usage_metric_)} => [:product_intelligence], %r{\A((ee|jh)/)?lib/gitlab/usage_data_counters/.*\.yml\z} => [:product_intelligence], %r{\A((ee|jh)/)?config/metrics/((.*\.yml)|(schema\.json))\z} => [:product_intelligence], %r{\A((ee|jh)/)?lib/gitlab/usage_data(_counters)?(/|\.rb)} => [:backend, :product_intelligence], @@ -108,9 +134,16 @@ module Tooling spec/lib/gitlab/tracking_spec\.rb | app/helpers/tracking_helper\.rb | spec/helpers/tracking_helper_spec\.rb | + (spec/)?lib/generators/gitlab/usage_metric_\S+ | + (spec/)?lib/generators/gitlab/usage_metric_definition/redis_hll_generator(_spec)?\.rb | lib/generators/rails/usage_metric_definition_generator\.rb | spec/lib/generators/usage_metric_definition_generator_spec\.rb | generator_templates/usage_metric_definition/metric_definition\.yml)\z}x => [:backend, :product_intelligence], + %r{gitlab/usage_data(_spec)?\.rb} => [:product_intelligence], + [%r{\.haml\z}, %r{data: \{ track}] => [:product_intelligence], + [%r{\.(rb|haml)\z}, %r{Gitlab::Tracking\.(event|enabled\?|options)$}] => [:product_intelligence], + [%r{\.(vue|js)\z}, %r{(Tracking.event|/\btrack\(/|data-track-action)}] => [:product_intelligence], + %r{\A((ee|jh)/)?app/(?!assets|views)[^/]+} => :backend, %r{\A((ee|jh)/)?(bin|config|generator_templates|lib|rubocop)/} => :backend, %r{\A((ee|jh)/)?spec/migrations} => :database, @@ -179,6 +212,10 @@ module Tooling read_file(filename).lines(chomp: true) end + def labels_to_add + @labels_to_add ||= [] + end + private def read_file(filename) @@ -187,7 +224,7 @@ module Tooling def ee? # Support former project name for `dev` and support local Danger run - %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?(File.expand_path('../../../ee', __dir__)) + %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?(File.expand_path('../../ee', __dir__)) end end end diff --git a/tooling/deprecations/docs.rb b/tooling/deprecations/docs.rb index 67ff7a932b4..0f649024b60 100644 --- a/tooling/deprecations/docs.rb +++ b/tooling/deprecations/docs.rb @@ -20,9 +20,11 @@ module Deprecations YAML.load_file(file) end - deprecations = VersionSorter.sort(deprecations) { |d| d["removal_milestone"] } + deps = VersionSorter.sort(deprecations) { |d| d["removal_milestone"] } - milestones = deprecations.map { |d| d["removal_milestone"] }.uniq + deprecations = deps.sort_by { |d| d["name"] } + + milestones = deps.map { |d| d["removal_milestone"] }.uniq template = Rails.root.join("data/deprecations/templates/_deprecation_template.md.erb") diff --git a/tooling/lib/tooling/images.rb b/tooling/lib/tooling/image.rb index d0c464b983c..d0c464b983c 100644 --- a/tooling/lib/tooling/images.rb +++ b/tooling/lib/tooling/image.rb diff --git a/tooling/quality/test_level.rb b/tooling/quality/test_level.rb index 83cbe7a1f19..5fbaad073c0 100644 --- a/tooling/quality/test_level.rb +++ b/tooling/quality/test_level.rb @@ -40,6 +40,7 @@ module Quality replicators routing rubocop + scripts serializers services sidekiq @@ -53,6 +54,7 @@ module Quality tooling ], integration: %w[ + commands controllers mailers requests diff --git a/tooling/rspec_flaky/report.rb b/tooling/rspec_flaky/report.rb index 3acfe7d2125..bde5115f03c 100644 --- a/tooling/rspec_flaky/report.rb +++ b/tooling/rspec_flaky/report.rb @@ -10,7 +10,7 @@ module RspecFlaky # This class is responsible for loading/saving JSON reports, and pruning # outdated examples. class Report < SimpleDelegator - OUTDATED_DAYS_THRESHOLD = 7 + OUTDATED_DAYS_THRESHOLD = 30 attr_reader :flaky_examples |