Add latest changes from gitlab-org/gitlab@14-6-stable-eev14.6.0-rc42

11 files changed, 172 insertions, 72 deletions

diff --git a/tooling/bin/find_changes b/tooling/bin/find_changes
index 20df085879a..c6b8bafbd85 100755
--- a/tooling/bin/find_changes
+++ b/tooling/bin/find_changes
@@ -3,19 +3,76 @@
 
 require 'gitlab'
 
-gitlab_token = ENV.fetch('PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE', '')
-gitlab_endpoint = ENV.fetch('CI_API_V4_URL')
-mr_project_path = ENV.fetch('CI_MERGE_REQUEST_PROJECT_PATH')
-mr_iid = ENV.fetch('CI_MERGE_REQUEST_IID')
+class FindChanges # rubocop:disable Gitlab/NamespacedClass
+  def initialize(output_file:, matched_tests_file: nil, frontend_fixtures_mapping_path: nil)
+    @gitlab_token = ENV.fetch('PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE', '')
+    @gitlab_endpoint = ENV.fetch('CI_API_V4_URL')
+    @mr_project_path = ENV.fetch('CI_MERGE_REQUEST_PROJECT_PATH')
+    @mr_iid = ENV.fetch('CI_MERGE_REQUEST_IID')
+    @output_file = output_file
+    @matched_tests_file = matched_tests_file
+    @frontend_fixtures_mapping_path = frontend_fixtures_mapping_path
+  end
 
-output_file = ARGV.shift
+  def execute
+    add_frontend_fixture_files!
+
+    File.write(output_file, file_changes.join(' '))
+  end
+
+  private
+
+  def add_frontend_fixture_files?
+    matched_tests_file && frontend_fixtures_mapping_path
+  end
+
+  def add_frontend_fixture_files!
+    return unless add_frontend_fixture_files?
+
+    # If we have a `test file -> JSON frontend fixture` mapping file, we add the files JSON frontend fixtures
+    # files to the list of changed files so that Jest can automatically run the dependent tests thanks to --findRelatedTests
+    test_files.each do |test_file|
+      file_changes.concat(frontend_fixtures_mapping[test_file]) if frontend_fixtures_mapping.key?(test_file)
+    end
+  end
+
+  def file_changes
+    @file_changes ||=
+      if File.exist?(output_file)
+        File.read(output_file).split(' ')
+      else
+        Gitlab.configure do |config|
+          config.endpoint      = gitlab_endpoint
+          config.private_token = gitlab_token
+        end
+
+        mr_changes = Gitlab.merge_request_changes(mr_project_path, mr_iid)
 
-Gitlab.configure do |config|
-  config.endpoint       = gitlab_endpoint
-  config.private_token  = gitlab_token
+        mr_changes.changes.map { |change| change['new_path'] }
+      end
+  end
+
+  def test_files
+    return [] if !matched_tests_file || !File.exist?(matched_tests_file)
+
+    File.read(matched_tests_file).split(' ')
+  end
+
+  def frontend_fixtures_mapping
+    return {} if !frontend_fixtures_mapping_path || !File.exist?(frontend_fixtures_mapping_path)
+
+    JSON.parse(File.read(frontend_fixtures_mapping_path)) # rubocop:disable Gitlab/Json
+  end
+
+  attr_reader :gitlab_token, :gitlab_endpoint, :mr_project_path, :mr_iid, :output_file, :matched_tests_file, :frontend_fixtures_mapping_path
 end
 
-mr_changes = Gitlab.merge_request_changes(mr_project_path, mr_iid)
-file_changes = mr_changes.changes.map { |change| change['new_path'] }
+output_file = ARGV.shift
+raise ArgumentError, "An path to an output file must be given as first argument of #{__FILE__}." if output_file.nil?
+
+matched_tests_file = ARGV.shift
+frontend_fixtures_mapping_path = ARGV.shift
 
-File.write(output_file, file_changes.join(' '))
+FindChanges
+  .new(output_file: output_file, matched_tests_file: matched_tests_file, frontend_fixtures_mapping_path: frontend_fixtures_mapping_path)
+  .execute
diff --git a/tooling/bin/qa/check_if_qa_only_spec_changes b/tooling/bin/qa/check_if_qa_only_spec_changes
new file mode 100755
index 00000000000..5b9166b41fe
--- /dev/null
+++ b/tooling/bin/qa/check_if_qa_only_spec_changes
@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# This script assumes the first argument is a path to a file containing a list of changed files and the second argument
+# is the path of a file where a list of end-to-end spec files with the leading 'qa/' trimmed will be written to if
+# all the files are end-to-end test spec files.
+
+abort("ERROR: Please specify the file containing the list of changed files and a file where the qa only spec files will be written") if ARGV.size != 2
+
+changed_files_path = ARGV.shift
+output_file = ARGV.shift
+
+return unless File.exist?(changed_files_path)
+
+changed_files = File.read(changed_files_path).split(' ')
+
+all_files_are_qa_specs = changed_files.all? { |file_path| file_path =~ %r{^qa\/qa\/specs\/features\/} }
+
+if all_files_are_qa_specs
+  qa_spec_paths_trimmed = changed_files.map { |path| path.sub('qa/', '') }
+  File.write(output_file, qa_spec_paths_trimmed.join(' '))
+end
diff --git a/tooling/danger/changelog.rb b/tooling/danger/changelog.rb
index fbf8ae931e2..6a392afac13 100644
--- a/tooling/danger/changelog.rb
+++ b/tooling/danger/changelog.rb
@@ -159,8 +159,8 @@ module Tooling
 
       def required_reasons
         [].tap do |reasons|
-          reasons << :db_changes if project_helper.changes.added.has_category?(:migration)
-          reasons << :feature_flag_removed if project_helper.changes.deleted.has_category?(:feature_flag)
+          reasons << :db_changes if helper.changes.added.has_category?(:migration)
+          reasons << :feature_flag_removed if helper.changes.deleted.has_category?(:feature_flag)
         end
       end
 
@@ -221,7 +221,7 @@ module Tooling
       end
 
       def categories_need_changelog?
-        (project_helper.changes.categories - NO_CHANGELOG_CATEGORIES).any?
+        (helper.changes.categories - NO_CHANGELOG_CATEGORIES).any?
       end
 
       def mr_without_no_changelog_label?
diff --git a/tooling/danger/product_intelligence.rb b/tooling/danger/product_intelligence.rb
index 72fc8deac43..6185b2f0d08 100644
--- a/tooling/danger/product_intelligence.rb
+++ b/tooling/danger/product_intelligence.rb
@@ -4,21 +4,32 @@
 module Tooling
   module Danger
     module ProductIntelligence
+      APPROVED_LABEL = 'product intelligence::approved'
+      REVIEW_LABEL = 'product intelligence::review pending'
+
       WORKFLOW_LABELS = [
-        'product intelligence::approved',
-        'product intelligence::review pending'
+        APPROVED_LABEL,
+        REVIEW_LABEL
       ].freeze
 
       def missing_labels
-        return [] if !helper.ci? || helper.mr_has_labels?('growth experiment')
+        return [] unless helper.ci?
 
         labels = []
         labels << 'product intelligence' unless helper.mr_has_labels?('product intelligence')
-        labels << 'product intelligence::review pending' unless has_workflow_labels?
+        labels << REVIEW_LABEL unless has_workflow_labels?
 
         labels
       end
 
+      def has_approved_label?
+        helper.mr_labels.include?(APPROVED_LABEL)
+      end
+
+      def skip_review?
+        helper.mr_has_labels?('growth experiment')
+      end
+
       private
 
       def has_workflow_labels?
diff --git a/tooling/danger/project_helper.rb b/tooling/danger/project_helper.rb
index 5d338393f90..b49df50c5f0 100644
--- a/tooling/danger/project_helper.rb
+++ b/tooling/danger/project_helper.rb
@@ -5,6 +5,7 @@ module Tooling
     module ProjectHelper
       LOCAL_RULES ||= %w[
         changelog
+        ci_config
         database
         documentation
         duplicate_yarn_dependencies
@@ -127,7 +128,7 @@ module Tooling
 
         %r{\A((spec/)?lib/generators/gitlab/usage_metric_)} => [:product_intelligence],
         %r{\A((ee|jh)/)?lib/gitlab/usage_data_counters/.*\.yml\z} => [:product_intelligence],
-        %r{\A((ee|jh)/)?config/metrics/((.*\.yml)|(schema\.json))\z} => [:product_intelligence],
+        %r{\A((ee|jh)/)?config/(events|metrics)/((.*\.yml)|(schema\.json))\z} => [:product_intelligence],
         %r{\A((ee|jh)/)?lib/gitlab/usage_data(_counters)?(/|\.rb)} => [:backend, :product_intelligence],
         %r{\A(
           lib/gitlab/tracking\.rb |
@@ -151,7 +152,8 @@ module Tooling
         %r{\A((ee|jh)/)?vendor/} => :backend,
         %r{\A(Gemfile|Gemfile.lock|Rakefile)\z} => :backend,
         %r{\A[A-Z_]+_VERSION\z} => :backend,
-        %r{\A\.rubocop((_manual)?_todo)?\.yml\z} => :backend,
+        %r{\A\.rubocop(_todo)?\.yml\z} => :backend,
+        %r{\A\.rubocop_todo/.*\.yml\z} => :backend,
         %r{\Afile_hooks/} => :backend,
 
         %r{\A((ee|jh)/)?qa/} => :qa,
@@ -174,18 +176,6 @@ module Tooling
         %r{\.js\z} => :frontend
       }.freeze
 
-      def changes_by_category
-        helper.changes_by_category(CATEGORIES)
-      end
-
-      def changes
-        helper.changes(CATEGORIES)
-      end
-
-      def categories_for_file(file)
-        helper.categories_for_file(file, CATEGORIES)
-      end
-
       def local_warning_message
         "#{MESSAGE_PREFIX} Only the following Danger rules can be run locally: #{LOCAL_RULES.join(', ')}"
       end
@@ -201,11 +191,7 @@ module Tooling
       end
 
       def all_ee_changes
-        changes.files.grep(%r{\Aee/})
-      end
-
-      def project_name
-        ee? ? 'gitlab' : 'gitlab-foss'
+        helper.changes.files.grep(%r{\Aee/})
       end
 
       def file_lines(filename)
@@ -221,11 +207,6 @@ module Tooling
       def read_file(filename)
         File.read(filename)
       end
-
-      def ee?
-        # Support former project name for `dev` and support local Danger run
-        %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?(File.expand_path('../../ee', __dir__))
-      end
     end
   end
 end
diff --git a/tooling/quality/test_level.rb b/tooling/quality/test_level.rb
index 5fbaad073c0..50cbc69beb2 100644
--- a/tooling/quality/test_level.rb
+++ b/tooling/quality/test_level.rb
@@ -33,6 +33,7 @@ module Quality
         initializers
         javascripts
         lib
+        metrics_server
         models
         policies
         presenters
@@ -44,6 +45,7 @@ module Quality
         serializers
         services
         sidekiq
+        sidekiq_cluster
         spam
         support_specs
         tasks
diff --git a/tooling/rspec_flaky/example.rb b/tooling/rspec_flaky/example.rb
index 18f8c5acc1c..e6c2f838194 100644
--- a/tooling/rspec_flaky/example.rb
+++ b/tooling/rspec_flaky/example.rb
@@ -38,6 +38,16 @@ module RspecFlaky
       rspec_example.respond_to?(:attempts) ? rspec_example.attempts : 1
     end
 
+    def to_h
+      {
+        example_id: example_id,
+        file: file,
+        line: line,
+        description: description,
+        last_attempts_count: attempts
+      }
+    end
+
     private
 
     attr_reader :rspec_example
diff --git a/tooling/rspec_flaky/flaky_example.rb b/tooling/rspec_flaky/flaky_example.rb
index 4f3688dbeed..299fcb567fc 100644
--- a/tooling/rspec_flaky/flaky_example.rb
+++ b/tooling/rspec_flaky/flaky_example.rb
@@ -3,38 +3,51 @@
 require 'ostruct'
 
 module RspecFlaky
+  ALLOWED_ATTRIBUTES = %i[
+    example_id
+    file
+    line
+    description
+    first_flaky_at
+    last_flaky_at
+    last_flaky_job
+    last_attempts_count
+    flaky_reports
+  ].freeze
+
   # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file
-  class FlakyExample < OpenStruct
-    def initialize(example)
-      if example.respond_to?(:example_id)
-        super(
-          example_id: example.example_id,
-          file: example.file,
-          line: example.line,
-          description: example.description,
-          last_attempts_count: example.attempts,
-          flaky_reports: 0)
-      else
-        super
+  class FlakyExample
+    def initialize(example_hash)
+      @attributes = {
+        first_flaky_at: Time.now,
+        last_flaky_at: Time.now,
+        last_flaky_job: nil,
+        last_attempts_count: example_hash[:attempts],
+        flaky_reports: 0
+      }.merge(example_hash.slice(*ALLOWED_ATTRIBUTES))
+
+      %i[first_flaky_at last_flaky_at].each do |attr|
+        attributes[attr] = Time.parse(attributes[attr]) if attributes[attr].is_a?(String)
       end
     end
 
     def update_flakiness!(last_attempts_count: nil)
-      self.first_flaky_at ||= Time.now
-      self.last_flaky_at = Time.now
-      self.flaky_reports += 1
-      self.last_attempts_count = last_attempts_count if last_attempts_count
+      attributes[:first_flaky_at] ||= Time.now
+      attributes[:last_flaky_at] = Time.now
+      attributes[:flaky_reports] += 1
+      attributes[:last_attempts_count] = last_attempts_count if last_attempts_count
 
-      if ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID']
-        self.last_flaky_job = "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}"
+      if ENV['CI_JOB_URL']
+        attributes[:last_flaky_job] = "#{ENV['CI_JOB_URL']}"
       end
     end
 
     def to_h
-      super.merge(
-        first_flaky_at: first_flaky_at,
-        last_flaky_at: last_flaky_at,
-        last_flaky_job: last_flaky_job)
+      attributes.dup
     end
+
+    private
+
+    attr_reader :attributes
   end
 end
diff --git a/tooling/rspec_flaky/flaky_examples_collection.rb b/tooling/rspec_flaky/flaky_examples_collection.rb
index acbfb411873..019ebf703da 100644
--- a/tooling/rspec_flaky/flaky_examples_collection.rb
+++ b/tooling/rspec_flaky/flaky_examples_collection.rb
@@ -16,7 +16,7 @@ module RspecFlaky
         collection.map do |uid, example|
           [
             uid,
-            example.is_a?(RspecFlaky::FlakyExample) ? example : RspecFlaky::FlakyExample.new(example)
+            RspecFlaky::FlakyExample.new(example.to_h.symbolize_keys)
           ]
         end
 
diff --git a/tooling/rspec_flaky/listener.rb b/tooling/rspec_flaky/listener.rb
index a5c68d830db..9b20eefc2f0 100644
--- a/tooling/rspec_flaky/listener.rb
+++ b/tooling/rspec_flaky/listener.rb
@@ -26,7 +26,7 @@ module RspecFlaky
 
       return unless current_example.attempts > 1
 
-      flaky_example = suite_flaky_examples.fetch(current_example.uid) { RspecFlaky::FlakyExample.new(current_example) }
+      flaky_example = suite_flaky_examples.fetch(current_example.uid) { RspecFlaky::FlakyExample.new(current_example.to_h) }
       flaky_example.update_flakiness!(last_attempts_count: current_example.attempts)
 
       flaky_examples[current_example.uid] = flaky_example
@@ -36,7 +36,6 @@ module RspecFlaky
       RspecFlaky::Report.new(flaky_examples).write(RspecFlaky::Config.flaky_examples_report_path)
       # write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path)
 
-      new_flaky_examples = flaky_examples - suite_flaky_examples
       if new_flaky_examples.any?
         rails_logger_warn("\nNew flaky examples detected:\n")
         rails_logger_warn(JSON.pretty_generate(new_flaky_examples.to_h)) # rubocop:disable Gitlab/Json
@@ -48,6 +47,10 @@ module RspecFlaky
 
     private
 
+    def new_flaky_examples
+      @new_flaky_examples ||= flaky_examples - suite_flaky_examples
+    end
+
     def init_suite_flaky_examples(suite_flaky_examples_json = nil)
       if suite_flaky_examples_json
         RspecFlaky::Report.load_json(suite_flaky_examples_json).flaky_examples
diff --git a/tooling/rspec_flaky/report.rb b/tooling/rspec_flaky/report.rb
index bde5115f03c..17dbb277446 100644
--- a/tooling/rspec_flaky/report.rb
+++ b/tooling/rspec_flaky/report.rb
@@ -10,7 +10,7 @@ module RspecFlaky
   # This class is responsible for loading/saving JSON reports, and pruning
   # outdated examples.
   class Report < SimpleDelegator
-    OUTDATED_DAYS_THRESHOLD = 30
+    OUTDATED_DAYS_THRESHOLD = 7
 
     attr_reader :flaky_examples
 
@@ -45,12 +45,13 @@ module RspecFlaky
 
     def prune_outdated(days: OUTDATED_DAYS_THRESHOLD)
       outdated_date_threshold = Time.now - (3600 * 24 * days)
-      updated_hash = flaky_examples.dup
-        .delete_if do |uid, hash|
-          hash[:last_flaky_at] && Time.parse(hash[:last_flaky_at]).to_i < outdated_date_threshold.to_i
+      recent_flaky_examples = flaky_examples.dup
+        .delete_if do |_uid, flaky_example|
+          last_flaky_at = flaky_example.to_h[:last_flaky_at]
+          last_flaky_at && last_flaky_at.to_i < outdated_date_threshold.to_i
         end
 
-      self.class.new(RspecFlaky::FlakyExamplesCollection.new(updated_hash))
+      self.class.new(RspecFlaky::FlakyExamplesCollection.new(recent_flaky_examples))
     end
   end
 end