Add latest changes from gitlab-org/gitlab@14-5-stable-eev14.5.0-rc42

10 files changed, 134 insertions, 61 deletions

diff --git a/tooling/bin/find_change_diffs b/tooling/bin/find_change_diffs
new file mode 100755
index 00000000000..7857945ea74
--- /dev/null
+++ b/tooling/bin/find_change_diffs
@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'gitlab'
+require 'pathname'
+
+# This script saves the diffs of changes in an MR to the directory specified as the first argument
+#
+# It exits with a success code if diffs are found and saved, or if there are no changes, including if the script runs in
+# a pipeline that is not for a merge request.
+
+gitlab_token = ENV.fetch('PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE')
+gitlab_endpoint = ENV.fetch('CI_API_V4_URL')
+mr_project_path = ENV['CI_MERGE_REQUEST_PROJECT_PATH']
+mr_iid = ENV['CI_MERGE_REQUEST_IID']
+
+puts "CI_MERGE_REQUEST_PROJECT_PATH is missing." if mr_project_path.to_s.empty?
+puts "CI_MERGE_REQUEST_IID is missing." if mr_iid.to_s.empty?
+
+unless mr_project_path && mr_iid
+  puts "Exiting as this does not appear to be a merge request pipeline."
+  exit
+end
+
+abort("ERROR: Please specify a directory to write MR diffs into.") if ARGV.empty?
+output_diffs_dir = Pathname.new(ARGV.shift).expand_path
+
+Gitlab.configure do |config|
+  config.endpoint       = gitlab_endpoint
+  config.private_token  = gitlab_token
+end
+
+Gitlab.merge_request_changes(mr_project_path, mr_iid).changes.each do |change|
+  next if change['diff'].empty?
+
+  output_diffs_dir.join(File.dirname(change['new_path'])).mkpath
+  output_diffs_dir.join("#{change['new_path']}.diff").write(change['diff'])
+end
diff --git a/tooling/bin/find_changes b/tooling/bin/find_changes
index 466510ccb19..20df085879a 100755
--- a/tooling/bin/find_changes
+++ b/tooling/bin/find_changes
@@ -3,7 +3,7 @@
 
 require 'gitlab'
 
-gitlab_token = ENV.fetch('DANGER_GITLAB_API_TOKEN', '')
+gitlab_token = ENV.fetch('PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE', '')
 gitlab_endpoint = ENV.fetch('CI_API_V4_URL')
 mr_project_path = ENV.fetch('CI_MERGE_REQUEST_PROJECT_PATH')
 mr_iid = ENV.fetch('CI_MERGE_REQUEST_IID')
diff --git a/tooling/bin/qa/package_and_qa_check b/tooling/bin/qa/package_and_qa_check
new file mode 100755
index 00000000000..21deb0fcd2d
--- /dev/null
+++ b/tooling/bin/qa/package_and_qa_check
@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'pathname'
+
+# This script checks if the package-and-qa job should trigger downstream pipelines to run the QA suite.
+#
+# It assumes the first argument is a directory of files containing diffs of changes from an MR
+# (e.g., created by tooling/bin/find_change_diffs). It exits with a success code if there are no diffs, or if the diffs
+# are suitable to run QA tests.
+#
+# The script will abort (exit code 1) if the argument is missing.
+#
+# The following condition will result in a failure code (2), indicating that package-and-qa should not run:
+#
+#   - If the changes only include tests being put in quarantine
+
+abort("ERROR: Please specify the directory containing MR diffs.") if ARGV.empty?
+diffs_dir = Pathname.new(ARGV.shift).expand_path
+
+# Run package-and-qa if there are no diffs. E.g., in scheduled pipelines
+exit 0 if diffs_dir.glob('**/*').empty?
+
+files_count = 0
+specs_count = 0
+quarantine_specs_count = 0
+
+diffs_dir.glob('**/*').each do |path|
+  next if path.directory?
+
+  files_count += 1
+  next unless path.to_s.end_with?('_spec.rb.diff')
+
+  specs_count += 1
+  quarantine_specs_count += 1 if path.read.match?(/^\+.*, quarantine:/)
+end
+
+# Run package-and-qa if there are no specs. E.g., when the MR changes QA framework files.
+exit 0 if specs_count == 0
+
+# Skip package-and-qa if there are only specs being put in quarantine.
+exit 2 if quarantine_specs_count == specs_count && quarantine_specs_count == files_count
+
+# Run package-and-qa under any other circumstances. E.g., if there are specs being put in quarantine but there are also
+# other changes that might need to be tested.
diff --git a/tooling/danger/changelog.rb b/tooling/danger/changelog.rb
index c053d366199..fbf8ae931e2 100644
--- a/tooling/danger/changelog.rb
+++ b/tooling/danger/changelog.rb
@@ -6,7 +6,7 @@ module Tooling
   module Danger
     module Changelog
       NO_CHANGELOG_LABELS = [
-        'tooling',
+        'type::tooling',
         'tooling::pipelines',
         'tooling::workflow',
         'ci-build',
diff --git a/tooling/danger/product_intelligence.rb b/tooling/danger/product_intelligence.rb
index 848f99eeff5..72fc8deac43 100644
--- a/tooling/danger/product_intelligence.rb
+++ b/tooling/danger/product_intelligence.rb
@@ -9,71 +9,20 @@ module Tooling
         'product intelligence::review pending'
       ].freeze
 
-      TRACKING_FILES = [
-        'lib/gitlab/tracking.rb',
-        'spec/lib/gitlab/tracking_spec.rb',
-        'app/helpers/tracking_helper.rb',
-        'spec/helpers/tracking_helper_spec.rb',
-        'app/assets/javascripts/tracking/index.js',
-        'app/assets/javascripts/tracking/constants.js',
-        'app/assets/javascripts/tracking/get_standard_context.js',
-        'spec/frontend/tracking/get_standard_context_spec.js',
-        'spec/frontend/tracking_spec.js',
-        'generator_templates/usage_metric_definition/metric_definition.yml',
-        'lib/generators/gitlab/usage_metric/usage_metric_generator.rb',
-        'lib/generators/gitlab/usage_metric_definition_generator.rb',
-        'lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb',
-        'spec/lib/generators/gitlab/usage_metric_generator_spec.rb',
-        'spec/lib/generators/gitlab/usage_metric_definition_generator_spec.rb',
-        'spec/lib/generators/gitlab/usage_metric_definition/redis_hll_generator_spec.rb',
-        'config/metrics/schema.json'
-      ].freeze
-
       def missing_labels
         return [] if !helper.ci? || helper.mr_has_labels?('growth experiment')
 
         labels = []
         labels << 'product intelligence' unless helper.mr_has_labels?('product intelligence')
-        labels << 'product intelligence::review pending' unless helper.mr_has_labels?(WORKFLOW_LABELS)
+        labels << 'product intelligence::review pending' unless has_workflow_labels?
 
         labels
       end
 
-      def matching_changed_files
-        tracking_changed_files = all_changed_files & TRACKING_FILES
-        usage_data_changed_files = all_changed_files.grep(%r{(usage_data)})
-
-        usage_data_changed_files + tracking_changed_files + metrics_changed_files + snowplow_changed_files
-      end
-
       private
 
-      def all_changed_files
-        helper.all_changed_files
-      end
-
-      def metrics_changed_files
-        all_changed_files.grep(%r{((ee/)?config/metrics/.*\.yml)})
-      end
-
-      def matching_files?(file, extension:, pattern:)
-        return unless file.end_with?(extension)
-
-        helper.changed_lines(file).grep(pattern).any?
-      end
-
-      def snowplow_changed_files
-        js_patterns = Regexp.union(
-          'Tracking.event',
-          /\btrack\(/,
-          'data-track-action'
-        )
-        all_changed_files.select do |file|
-          matching_files?(file, extension: '.rb', pattern: %r{Gitlab::Tracking\.(event|enabled\?|options)$}) ||
-            matching_files?(file, extension: '.js', pattern: js_patterns) ||
-            matching_files?(file, extension: '.vue', pattern: js_patterns) ||
-            matching_files?(file, extension: '.haml', pattern: %r{data: \{ track})
-        end
+      def has_workflow_labels?
+        (WORKFLOW_LABELS & helper.mr_labels).any?
       end
     end
   end
diff --git a/tooling/danger/project_helper.rb b/tooling/danger/project_helper.rb
index c552a75bba8..5d338393f90 100644
--- a/tooling/danger/project_helper.rb
+++ b/tooling/danger/project_helper.rb
@@ -22,12 +22,12 @@ module Tooling
         ce_ee_vue_templates
         ci_templates
         datateam
-        metadata
         feature_flag
         roulette
         sidekiq_queues
         specialization_labels
         specs
+        z_metadata
       ].freeze
 
       MESSAGE_PREFIX = '==>'
@@ -38,10 +38,34 @@ module Tooling
 
         %r{\A((ee|jh)/)?config/feature_flags/} => :feature_flag,
 
+        %r{doc/api/usage_data.md} => [:product_intelligence],
+
         %r{\Adoc/.*(\.(md|png|gif|jpg|yml))\z} => :docs,
         %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs,
         %r{\Adata/whats_new/} => :docs,
 
+        %r{\A((ee|jh)/)?app/finders/(.+/)?integrations/} => [:integrations_be, :database, :backend],
+        [%r{\A((ee|jh)/)?db/(geo/)?(migrate|post_migrate)/}, %r{(:integrations|:\w+_tracker_data)\b}] => [:integrations_be, :database, :migration],
+        [%r{\A((ee|jh)/)?(app|lib)/.+\.rb}, %r{\b(Integrations::|\.execute_(integrations|hooks))\b}] => [:integrations_be, :backend],
+        %r{\A(
+          ((ee|jh)/)?app/((?!.*clusters)(?!.*alert_management)(?!.*views)(?!.*assets).+/)?integration.+ |
+          ((ee|jh)/)?app/((?!.*search).+/)?project_service.+ |
+          ((ee|jh)/)?app/(models|helpers|workers|services|controllers)/(.+/)?(jira_connect.+|.*hook.+) |
+          ((ee|jh)/)?app/controllers/(.+/)?oauth/jira/.+ |
+          ((ee|jh)/)?app/services/(.+/)?jira.+ |
+          ((ee|jh)/)?app/workers/(.+/)?(propagate_integration.+|irker_worker\.rb) |
+          ((ee|jh)/)?lib/(.+/)?(atlassian|data_builder|hook_data)/.+ |
+          ((ee|jh)/)?lib/(.+/)?.*integration.+ |
+          ((ee|jh)/)?lib/(.+/)?api/v3/github\.rb |
+          ((ee|jh)/)?lib/(.+/)?api/github/entities\.rb
+        )\z}x => [:integrations_be, :backend],
+
+        %r{\A(
+          ((ee|jh)/)?app/(views|assets)/((?!.*clusters)(?!.*alerts_settings).+/)?integration.+ |
+          ((ee|jh)/)?app/(views|assets)/(.+/)?jira_connect.+ |
+          ((ee|jh)/)?app/(views|assets)/((?!.*filtered_search).+/)?hooks?.+
+        )\z}x => [:integrations_fe, :frontend],
+
         %r{\A(
           app/assets/javascripts/tracking/.*\.js |
           spec/frontend/tracking/.*\.js |
@@ -82,6 +106,7 @@ module Tooling
         %r{\A((ee|jh)/)?app/finders/} => [:database, :backend],
         %r{\Arubocop/cop/migration(/|\.rb)} => :database,
 
+        %r{\A(\.ruby-version\z|\.nvmrc\z|\.tool-versions\z)} => :tooling,
         %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :tooling,
         %r{\A\.codeclimate\.yml\z} => :tooling,
         %r{\Alefthook.yml\z} => :tooling,
@@ -100,6 +125,7 @@ module Tooling
         %r{\A((ee|jh)/)?spec/support/shared_contexts/features/} => :test,
         %r{\A((ee|jh)/)?spec/support/helpers/features/} => :test,
 
+        %r{\A((spec/)?lib/generators/gitlab/usage_metric_)} => [:product_intelligence],
         %r{\A((ee|jh)/)?lib/gitlab/usage_data_counters/.*\.yml\z} => [:product_intelligence],
         %r{\A((ee|jh)/)?config/metrics/((.*\.yml)|(schema\.json))\z} => [:product_intelligence],
         %r{\A((ee|jh)/)?lib/gitlab/usage_data(_counters)?(/|\.rb)} => [:backend, :product_intelligence],
@@ -108,9 +134,16 @@ module Tooling
           spec/lib/gitlab/tracking_spec\.rb |
           app/helpers/tracking_helper\.rb |
           spec/helpers/tracking_helper_spec\.rb |
+          (spec/)?lib/generators/gitlab/usage_metric_\S+ |
+          (spec/)?lib/generators/gitlab/usage_metric_definition/redis_hll_generator(_spec)?\.rb |
           lib/generators/rails/usage_metric_definition_generator\.rb |
           spec/lib/generators/usage_metric_definition_generator_spec\.rb |
           generator_templates/usage_metric_definition/metric_definition\.yml)\z}x => [:backend, :product_intelligence],
+        %r{gitlab/usage_data(_spec)?\.rb} => [:product_intelligence],
+        [%r{\.haml\z}, %r{data: \{ track}] => [:product_intelligence],
+        [%r{\.(rb|haml)\z}, %r{Gitlab::Tracking\.(event|enabled\?|options)$}] => [:product_intelligence],
+        [%r{\.(vue|js)\z}, %r{(Tracking.event|/\btrack\(/|data-track-action)}] => [:product_intelligence],
+
         %r{\A((ee|jh)/)?app/(?!assets|views)[^/]+} => :backend,
         %r{\A((ee|jh)/)?(bin|config|generator_templates|lib|rubocop)/} => :backend,
         %r{\A((ee|jh)/)?spec/migrations} => :database,
@@ -179,6 +212,10 @@ module Tooling
         read_file(filename).lines(chomp: true)
       end
 
+      def labels_to_add
+        @labels_to_add ||= []
+      end
+
       private
 
       def read_file(filename)
@@ -187,7 +224,7 @@ module Tooling
 
       def ee?
         # Support former project name for `dev` and support local Danger run
-        %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?(File.expand_path('../../../ee', __dir__))
+        %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?(File.expand_path('../../ee', __dir__))
       end
     end
   end
diff --git a/tooling/deprecations/docs.rb b/tooling/deprecations/docs.rb
index 67ff7a932b4..0f649024b60 100644
--- a/tooling/deprecations/docs.rb
+++ b/tooling/deprecations/docs.rb
@@ -20,9 +20,11 @@ module Deprecations
         YAML.load_file(file)
       end
 
-      deprecations = VersionSorter.sort(deprecations) { |d| d["removal_milestone"] }
+      deps = VersionSorter.sort(deprecations) { |d| d["removal_milestone"] }
 
-      milestones = deprecations.map { |d| d["removal_milestone"] }.uniq
+      deprecations = deps.sort_by { |d| d["name"] }
+
+      milestones = deps.map { |d| d["removal_milestone"] }.uniq
 
       template = Rails.root.join("data/deprecations/templates/_deprecation_template.md.erb")
 
diff --git a/tooling/lib/tooling/images.rb b/tooling/lib/tooling/image.rb
index d0c464b983c..d0c464b983c 100644
--- a/tooling/lib/tooling/images.rb
+++ b/tooling/lib/tooling/image.rb
diff --git a/tooling/quality/test_level.rb b/tooling/quality/test_level.rb
index 83cbe7a1f19..5fbaad073c0 100644
--- a/tooling/quality/test_level.rb
+++ b/tooling/quality/test_level.rb
@@ -40,6 +40,7 @@ module Quality
         replicators
         routing
         rubocop
+        scripts
         serializers
         services
         sidekiq
@@ -53,6 +54,7 @@ module Quality
         tooling
       ],
       integration: %w[
+        commands
         controllers
         mailers
         requests
diff --git a/tooling/rspec_flaky/report.rb b/tooling/rspec_flaky/report.rb
index 3acfe7d2125..bde5115f03c 100644
--- a/tooling/rspec_flaky/report.rb
+++ b/tooling/rspec_flaky/report.rb
@@ -10,7 +10,7 @@ module RspecFlaky
   # This class is responsible for loading/saving JSON reports, and pruning
   # outdated examples.
   class Report < SimpleDelegator
-    OUTDATED_DAYS_THRESHOLD = 7
+    OUTDATED_DAYS_THRESHOLD = 30
 
     attr_reader :flaky_examples