Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43

7 files changed, 264 insertions, 19 deletions

diff --git a/tooling/danger/changelog.rb b/tooling/danger/changelog.rb
index 672d23d58e4..065c737050e 100644
--- a/tooling/danger/changelog.rb
+++ b/tooling/danger/changelog.rb
@@ -18,29 +18,35 @@ module Tooling
       CHANGELOG_MODIFIED_URL_TEXT = "**CHANGELOG.md was edited.** Please remove the additions and create a CHANGELOG entry.\n\n"
       CHANGELOG_MISSING_URL_TEXT = "**[CHANGELOG missing](https://docs.gitlab.com/ee/development/changelog.html)**:\n\n"
 
-      OPTIONAL_CHANGELOG_MESSAGE = <<~MSG
-      If you want to create a changelog entry for GitLab FOSS, run the following:
+      OPTIONAL_CHANGELOG_MESSAGE = {
+        local: "If this merge request [doesn't need a CHANGELOG entry](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry), feel free to ignore this message.",
+        ci: <<~MSG
+        If you want to create a changelog entry for GitLab FOSS, run the following:
 
-          #{CREATE_CHANGELOG_COMMAND}
+            #{CREATE_CHANGELOG_COMMAND}
 
-      If you want to create a changelog entry for GitLab EE, run the following instead:
+        If you want to create a changelog entry for GitLab EE, run the following instead:
 
-          #{CREATE_EE_CHANGELOG_COMMAND}
+            #{CREATE_EE_CHANGELOG_COMMAND}
 
-      If this merge request [doesn't need a CHANGELOG entry](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry), feel free to ignore this message.
-      MSG
+        If this merge request [doesn't need a CHANGELOG entry](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry), feel free to ignore this message.
+        MSG
+      }.freeze
 
       REQUIRED_CHANGELOG_REASONS = {
         db_changes: 'introduces a database migration',
         feature_flag_removed: 'removes a feature flag'
       }.freeze
-      REQUIRED_CHANGELOG_MESSAGE = <<~MSG
-      To create a changelog entry, run the following:
+      REQUIRED_CHANGELOG_MESSAGE = {
+        local: "This merge request requires a changelog entry because it [%<reason>s](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry).",
+        ci: <<~MSG
+        To create a changelog entry, run the following:
 
-          #{CREATE_CHANGELOG_COMMAND}
+            #{CREATE_CHANGELOG_COMMAND}
 
-      This merge request requires a changelog entry because it [%<reason>s](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry).
-      MSG
+        This merge request requires a changelog entry because it [%<reason>s](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry).
+        MSG
+      }.freeze
 
       def required_reasons
         [].tap do |reasons|
@@ -67,20 +73,20 @@ module Tooling
 
       def modified_text
         CHANGELOG_MODIFIED_URL_TEXT +
-          format(OPTIONAL_CHANGELOG_MESSAGE, mr_iid: helper.mr_iid, mr_title: sanitized_mr_title)
+          (helper.ci? ? format(OPTIONAL_CHANGELOG_MESSAGE[:ci], mr_iid: helper.mr_iid, mr_title: sanitized_mr_title) : OPTIONAL_CHANGELOG_MESSAGE[:local])
       end
 
       def required_texts
         required_reasons.each_with_object({}) do |required_reason, memo|
           memo[required_reason] =
             CHANGELOG_MISSING_URL_TEXT +
-              format(REQUIRED_CHANGELOG_MESSAGE, reason: REQUIRED_CHANGELOG_REASONS.fetch(required_reason), mr_iid: helper.mr_iid, mr_title: sanitized_mr_title)
+              (helper.ci? ? format(REQUIRED_CHANGELOG_MESSAGE[:ci], reason: REQUIRED_CHANGELOG_REASONS.fetch(required_reason), mr_iid: helper.mr_iid, mr_title: sanitized_mr_title) : REQUIRED_CHANGELOG_MESSAGE[:local])
         end
       end
 
       def optional_text
         CHANGELOG_MISSING_URL_TEXT +
-          format(OPTIONAL_CHANGELOG_MESSAGE, mr_iid: helper.mr_iid, mr_title: sanitized_mr_title)
+          (helper.ci? ? format(OPTIONAL_CHANGELOG_MESSAGE[:ci], mr_iid: helper.mr_iid, mr_title: sanitized_mr_title) : OPTIONAL_CHANGELOG_MESSAGE[:local])
       end
 
       private
diff --git a/tooling/danger/project_helper.rb b/tooling/danger/project_helper.rb
index 4458000261f..7aac0de0b6b 100644
--- a/tooling/danger/project_helper.rb
+++ b/tooling/danger/project_helper.rb
@@ -4,9 +4,11 @@ module Tooling
   module Danger
     module ProjectHelper
       LOCAL_RULES ||= %w[
+        changelog
         changes_size
         commit_messages
         database
+        datateam
         documentation
         duplicate_yarn_dependencies
         eslint
@@ -20,7 +22,6 @@ module Tooling
 
       CI_ONLY_RULES ||= %w[
         ce_ee_vue_templates
-        changelog
         ci_templates
         metadata
         feature_flag
@@ -30,20 +31,25 @@ module Tooling
         specs
       ].freeze
 
-      MESSAGE_PREFIX = '==>'.freeze
+      MESSAGE_PREFIX = '==>'
 
       # First-match win, so be sure to put more specific regex at the top...
       CATEGORIES = {
-        [%r{usage_data\.rb}, %r{^(\+|-).*\s+(count|distinct_count|estimate_batch_distinct_count)\(.*\)(.*)$}] => [:database, :backend],
+        [%r{usage_data\.rb}, %r{^(\+|-).*\s+(count|distinct_count|estimate_batch_distinct_count)\(.*\)(.*)$}] => [:database, :backend, :product_intelligence],
 
         %r{\A(ee/)?config/feature_flags/} => :feature_flag,
 
         %r{\A(ee/)?(changelogs/unreleased)(-ee)?/} => :changelog,
 
+        %r{\Adoc/development/usage_ping/dictionary\.md\z} => [:docs, :product_intelligence],
         %r{\Adoc/.*(\.(md|png|gif|jpg))\z} => :docs,
         %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs,
         %r{\Adata/whats_new/} => :docs,
 
+        %r{\A(
+          app/assets/javascripts/tracking\.js |
+          spec/frontend/tracking_spec\.js
+        )\z}x => [:frontend, :product_intelligence],
         %r{\A(ee/)?app/(assets|views)/} => :frontend,
         %r{\A(ee/)?public/} => :frontend,
         %r{\A(ee/)?spec/(javascripts|frontend)/} => :frontend,
@@ -73,6 +79,7 @@ module Tooling
         %r{\A(ee/)?db/(geo/)?(migrate|post_migrate)/} => [:database, :migration],
         %r{\A(ee/)?db/(?!fixtures)[^/]+} => :database,
         %r{\A(ee/)?lib/gitlab/(database|background_migration|sql|github_import)(/|\.rb)} => :database,
+        %r{\A(app/services/authorized_project_update/find_records_due_for_refresh_service)(/|\.rb)} => :database,
         %r{\A(app/models/project_authorization|app/services/users/refresh_authorized_projects_service)(/|\.rb)} => :database,
         %r{\A(ee/)?app/finders/} => :database,
         %r{\Arubocop/cop/migration(/|\.rb)} => :database,
@@ -95,6 +102,17 @@ module Tooling
         %r{\A(ee/)?spec/support/shared_contexts/features/} => :test,
         %r{\A(ee/)?spec/support/helpers/features/} => :test,
 
+        %r{\A(ee/)?lib/gitlab/usage_data_counters/.*\.yml\z} => [:product_intelligence],
+        %r{\A(ee/)?config/metrics/((.*\.yml)|(schema\.json))\z} => [:product_intelligence],
+        %r{\A(ee/)?lib/gitlab/usage_data(_counters)?(/|\.rb)} => [:backend, :product_intelligence],
+        %r{\A(
+          lib/gitlab/tracking\.rb |
+          spec/lib/gitlab/tracking_spec\.rb |
+          app/helpers/tracking_helper\.rb |
+          spec/helpers/tracking_helper_spec\.rb |
+          lib/generators/rails/usage_metric_definition_generator\.rb |
+          spec/lib/generators/usage_metric_definition_generator_spec\.rb |
+          generator_templates/usage_metric_definition/metric_definition\.yml)\z}x => [:backend, :product_intelligence],
         %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend,
         %r{\A(ee/)?(bin|config|generator_templates|lib|rubocop)/} => :backend,
         %r{\A(ee/)?spec/} => :backend,
@@ -106,6 +124,8 @@ module Tooling
 
         %r{\A(ee/)?qa/} => :qa,
 
+        %r{\Aworkhorse/.*} => :workhorse,
+
         # Files that don't fit into any category are marked with :none
         %r{\A(ee/)?changelogs/} => :none,
         %r{\Alocale/gitlab\.pot\z} => :none,
diff --git a/tooling/lib/tooling/kubernetes_client.rb b/tooling/lib/tooling/kubernetes_client.rb
index 35605fd493c..9bc5626db6b 100644
--- a/tooling/lib/tooling/kubernetes_client.rb
+++ b/tooling/lib/tooling/kubernetes_client.rb
@@ -22,6 +22,8 @@ module Tooling
 
     def cleanup_by_created_at(resource_type:, created_before:, wait: true)
       resource_names = resource_names_created_before(resource_type: resource_type, created_before: created_before)
+      return if resource_names.empty?
+
       delete_by_exact_names(resource_type: resource_type, resource_names: resource_names, wait: wait)
     end
 
diff --git a/tooling/lib/tooling/test_map_packer.rb b/tooling/lib/tooling/test_map_packer.rb
index 520d69610eb..d74edb9500f 100644
--- a/tooling/lib/tooling/test_map_packer.rb
+++ b/tooling/lib/tooling/test_map_packer.rb
@@ -2,7 +2,7 @@
 
 module Tooling
   class TestMapPacker
-    SEPARATOR = '/'.freeze
+    SEPARATOR = '/'
     MARKER = 1
 
     def pack(map)
diff --git a/tooling/merge_request.rb b/tooling/merge_request.rb
new file mode 100644
index 00000000000..d0f32a611aa
--- /dev/null
+++ b/tooling/merge_request.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'faraday_middleware'
+
+module Tooling
+  class MergeRequest
+    GITLAB_API_URL_TEMPLATE = 'https://gitlab.com/api/v4/projects/%{project_path}/merge_requests'
+
+    def self.for(branch:, project_path:)
+      url = format(GITLAB_API_URL_TEMPLATE, { project_path: URI.encode_www_form_component(project_path) })
+
+      conn = Faraday.new(url) do |conn|
+        conn.request :json
+        conn.response :json, content_type: /\bjson$/
+        conn.adapter Faraday.default_adapter
+      end
+
+      response = conn.get do |req|
+        req.params[:source_branch] = branch
+        req.params[:order_by] = 'updated_at'
+        req.params[:sort] = 'desc'
+      end
+
+      new(response.body.first)
+    end
+
+    attr_reader :merge_request
+
+    def initialize(merge_request)
+      @merge_request = merge_request
+    end
+
+    def iid
+      merge_request['iid']
+    end
+  end
+end
diff --git a/tooling/merge_request_rspec_failure_rake_task.rb b/tooling/merge_request_rspec_failure_rake_task.rb
new file mode 100644
index 00000000000..02ac38bee69
--- /dev/null
+++ b/tooling/merge_request_rspec_failure_rake_task.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'test_file_finder'
+require_relative './merge_request'
+
+module Tooling
+  class MergeRequestRspecFailureRakeTask < RSpec::Core::RakeTask
+    PROJECT_PATH = 'gitlab-org/gitlab'
+
+    def run_task(_verbose)
+      if pattern.empty?
+        puts "No rspec failures in the merge request."
+        return
+      end
+
+      super
+    end
+
+    def rspec_failures_on_merge_request
+      test_file_finder = TestFileFinder::FileFinder.new
+      test_file_finder.use TestFileFinder::MappingStrategies::GitlabMergeRequestRspecFailure.new(project_path: PROJECT_PATH, merge_request_iid: merge_request.iid)
+      test_file_finder.test_files
+    rescue TestFileFinder::TestReportError => e
+      abort e.message
+    end
+
+    private
+
+    def merge_request
+      @merge_request ||= Tooling::MergeRequest.for(branch: current_branch, project_path: PROJECT_PATH)
+    end
+
+    def current_branch
+      @current_branch ||= `git branch --show-current`.strip
+    end
+  end
+end
diff --git a/tooling/quality/test_level.rb b/tooling/quality/test_level.rb
new file mode 100644
index 00000000000..ad9de067375
--- /dev/null
+++ b/tooling/quality/test_level.rb
@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module Quality
+  class TestLevel
+    UnknownTestLevelError = Class.new(StandardError)
+
+    TEST_LEVEL_FOLDERS = {
+      migration: %w[
+        migrations
+      ],
+      background_migration: %w[
+        lib/gitlab/background_migration
+        lib/ee/gitlab/background_migration
+      ],
+      frontend_fixture: %w[
+        frontend/fixtures
+      ],
+      unit: %w[
+        bin
+        channels
+        config
+        db
+        dependencies
+        elastic
+        elastic_integration
+        experiments
+        factories
+        finders
+        frontend
+        graphql
+        haml_lint
+        helpers
+        initializers
+        javascripts
+        lib
+        models
+        policies
+        presenters
+        rack_servers
+        replicators
+        routing
+        rubocop
+        serializers
+        services
+        sidekiq
+        spam
+        support_specs
+        tasks
+        uploaders
+        validators
+        views
+        workers
+        tooling
+      ],
+      integration: %w[
+        controllers
+        mailers
+        requests
+      ],
+      system: ['features']
+    }.freeze
+
+    attr_reader :prefix
+
+    def initialize(prefix = nil)
+      @prefix = prefix
+      @patterns = {}
+      @regexps = {}
+    end
+
+    def pattern(level)
+      @patterns[level] ||= "#{prefix}spec/#{folders_pattern(level)}{,/**/}*#{suffix(level)}"
+    end
+
+    def regexp(level)
+      @regexps[level] ||= Regexp.new("#{prefix}spec/#{folders_regex(level)}").freeze
+    end
+
+    def level_for(file_path)
+      case file_path
+      # Detect migration first since some background migration tests are under
+      # spec/lib/gitlab/background_migration and tests under spec/lib are unit by default
+      when regexp(:migration), regexp(:background_migration)
+        :migration
+      # Detect frontend fixture before matching other unit tests
+      when regexp(:frontend_fixture)
+        :frontend_fixture
+      when regexp(:unit)
+        :unit
+      when regexp(:integration)
+        :integration
+      when regexp(:system)
+        :system
+      else
+        raise UnknownTestLevelError, "Test level for #{file_path} couldn't be set. Please rename the file properly or change the test level detection regexes in #{__FILE__}."
+      end
+    end
+
+    def background_migration?(file_path)
+      !!(file_path =~ regexp(:background_migration))
+    end
+
+    private
+
+    def suffix(level)
+      case level
+      when :frontend_fixture
+        ".rb"
+      else
+        "_spec.rb"
+      end
+    end
+
+    def migration_and_background_migration_folders
+      TEST_LEVEL_FOLDERS.fetch(:migration) + TEST_LEVEL_FOLDERS.fetch(:background_migration)
+    end
+
+    def folders_pattern(level)
+      case level
+      when :migration
+        "{#{migration_and_background_migration_folders.join(',')}}"
+      # Geo specs aren't in a specific folder, but they all have the :geo tag, so we must search for them globally
+      when :all, :geo
+        '**'
+      else
+        "{#{TEST_LEVEL_FOLDERS.fetch(level).join(',')}}"
+      end
+    end
+
+    def folders_regex(level)
+      case level
+      when :migration
+        "(#{migration_and_background_migration_folders.join('|')})"
+      # Geo specs aren't in a specific folder, but they all have the :geo tag, so we must search for them globally
+      when :all, :geo
+        ''
+      else
+        "(#{TEST_LEVEL_FOLDERS.fetch(level).join('|')})"
+      end
+    end
+  end
+end