Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42

9 files changed, 471 insertions, 11 deletions

diff --git a/rubocop/code_reuse_helpers.rb b/rubocop/code_reuse_helpers.rb
index 63019c43943..283c43de227 100644
--- a/rubocop/code_reuse_helpers.rb
+++ b/rubocop/code_reuse_helpers.rb
@@ -32,59 +32,79 @@ module RuboCop
 
     # Returns true if the given node resides in app/finders or ee/app/finders.
     def in_finder?(node)
-      in_directory?(node, 'finders')
+      in_app_directory?(node, 'finders')
     end
 
     # Returns true if the given node resides in app/models or ee/app/models.
     def in_model?(node)
-      in_directory?(node, 'models')
+      in_app_directory?(node, 'models')
     end
 
     # Returns true if the given node resides in app/services or ee/app/services.
     def in_service_class?(node)
-      in_directory?(node, 'services')
+      in_app_directory?(node, 'services')
     end
 
     # Returns true if the given node resides in app/presenters or
     # ee/app/presenters.
     def in_presenter?(node)
-      in_directory?(node, 'presenters')
+      in_app_directory?(node, 'presenters')
     end
 
     # Returns true if the given node resides in app/serializers or
     # ee/app/serializers.
     def in_serializer?(node)
-      in_directory?(node, 'serializers')
+      in_app_directory?(node, 'serializers')
     end
 
     # Returns true if the given node resides in app/workers or ee/app/workers.
     def in_worker?(node)
-      in_directory?(node, 'workers')
+      in_app_directory?(node, 'workers')
     end
 
     # Returns true if the given node resides in app/controllers or
     # ee/app/controllers.
     def in_controller?(node)
-      in_directory?(node, 'controllers')
+      in_app_directory?(node, 'controllers')
+    end
+
+    # Returns true if the given node resides in app/graphql/types,
+    # ee/app/graphql/types, or ee/app/graphql/ee/types.
+    def in_graphql_types?(node)
+      in_app_directory?(node, 'graphql/types') || in_app_directory?(node, 'graphql/ee/types')
     end
 
     # Returns true if the given node resides in lib/api or ee/lib/api.
     def in_api?(node)
+      in_lib_directory?(node, 'api')
+    end
+
+    # Returns true if the given node resides in spec or ee/spec.
+    def in_spec?(node)
       file_path_for_node(node).start_with?(
-        File.join(ce_lib_directory, 'api'),
-        File.join(ee_lib_directory, 'api')
+        ce_spec_directory,
+        ee_spec_directory
       )
     end
 
     # Returns `true` if the given AST node resides in the given directory,
     # relative to app and/or ee/app.
-    def in_directory?(node, directory)
+    def in_app_directory?(node, directory)
       file_path_for_node(node).start_with?(
         File.join(ce_app_directory, directory),
         File.join(ee_app_directory, directory)
       )
     end
 
+    # Returns `true` if the given AST node resides in the given directory,
+    # relative to lib and/or ee/lib.
+    def in_lib_directory?(node, directory)
+      file_path_for_node(node).start_with?(
+        File.join(ce_lib_directory, directory),
+        File.join(ee_lib_directory, directory)
+      )
+    end
+
     # Returns the receiver name of a send node.
     #
     # For the AST node `(send (const nil? :Foo) ...)` this would return
@@ -149,6 +169,14 @@ module RuboCop
       File.join(rails_root, 'ee', 'lib')
     end
 
+    def ce_spec_directory
+      File.join(rails_root, 'spec')
+    end
+
+    def ee_spec_directory
+      File.join(rails_root, 'ee', 'spec')
+    end
+
     def rails_root
       File.expand_path('..', __dir__)
     end
diff --git a/rubocop/cop/database/multiple_databases.rb b/rubocop/cop/database/multiple_databases.rb
new file mode 100644
index 00000000000..fb6e81f9845
--- /dev/null
+++ b/rubocop/cop/database/multiple_databases.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module RuboCop
+  module Cop
+    module Database
+      # @example
+      #   # bad
+      #   ActiveRecord::Base.connection
+      #
+      #   # good
+      #   ApplicationRecord.connection
+      #
+      class MultipleDatabases < RuboCop::Cop::Cop
+        AR_BASE_MESSAGE = <<~EOF
+          Do not use methods from ActiveRecord::Base, use the ApplicationRecord class instead
+          For fixing offenses related to the ActiveRecord::Base.transaction method, see our guidelines:
+          https://docs.gitlab.com/ee/development/database/transaction_guidelines.html
+        EOF
+
+        def_node_matcher :active_record_base_method_is_used?, <<~PATTERN
+        (send (const (const nil? :ActiveRecord) :Base) $_)
+        PATTERN
+
+        def on_send(node)
+          return unless active_record_base_method_is_used?(node)
+
+          add_offense(node, location: :expression, message: AR_BASE_MESSAGE)
+        end
+      end
+    end
+  end
+end
diff --git a/rubocop/cop/gitlab/duplicate_spec_location.rb b/rubocop/cop/gitlab/duplicate_spec_location.rb
index 025f75d644f..ece3b9313d9 100644
--- a/rubocop/cop/gitlab/duplicate_spec_location.rb
+++ b/rubocop/cop/gitlab/duplicate_spec_location.rb
@@ -25,7 +25,7 @@ module RuboCop
         MSG = 'Duplicate spec location in `%<path>s`.'
 
         def on_top_level_describe(node, _args)
-          path = file_path_for_node(node).sub(/\A#{rails_root}\//, '')
+          path = file_path_for_node(node).sub(%r{\A#{rails_root}/}, '')
           duplicate_path = find_duplicate_path(path)
 
           if duplicate_path && File.exist?(File.join(rails_root, duplicate_path))
diff --git a/rubocop/cop/gitlab/mark_used_feature_flags.rb b/rubocop/cop/gitlab/mark_used_feature_flags.rb
new file mode 100644
index 00000000000..2a020d6efb2
--- /dev/null
+++ b/rubocop/cop/gitlab/mark_used_feature_flags.rb
@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+require_relative '../../code_reuse_helpers'
+
+module RuboCop
+  module Cop
+    module Gitlab
+      # This cop tracks the usage of feature flags among the codebase.
+      #
+      # The files set in `tmp/feature_flags/*.used` can then be used for verification purpose.
+      #
+      class MarkUsedFeatureFlags < RuboCop::Cop::Cop
+        include RuboCop::CodeReuseHelpers
+
+        FEATURE_METHODS = %i[enabled? disabled?].freeze
+        EXPERIMENTATION_METHODS = %i[active?].freeze
+        EXPERIMENT_METHODS = %i[
+          experiment
+          experiment_enabled?
+          push_frontend_experiment
+        ].freeze
+        RUGGED_METHODS = %i[
+          use_rugged?
+        ].freeze
+        WORKER_METHODS = %i[
+          data_consistency
+          deduplicate
+        ].freeze
+        GRAPHQL_METHODS = %i[
+          field
+        ].freeze
+        SELF_METHODS = %i[
+          push_frontend_feature_flag
+          limit_feature_flag=
+        ].freeze + EXPERIMENT_METHODS + RUGGED_METHODS + WORKER_METHODS
+
+        RESTRICT_ON_SEND = FEATURE_METHODS + EXPERIMENTATION_METHODS + GRAPHQL_METHODS + SELF_METHODS
+
+        USAGE_DATA_COUNTERS_EVENTS_YAML_GLOBS = [
+          File.expand_path("../../../config/metrics/aggregates/*.yml", __dir__),
+          File.expand_path("../../../lib/gitlab/usage_data_counters/known_events/*.yml", __dir__)
+        ].freeze
+
+        DYNAMIC_FEATURE_FLAGS = [
+          :usage_data_static_site_editor_commits, # https://gitlab.com/gitlab-org/gitlab/-/issues/284082
+          :usage_data_static_site_editor_merge_requests # https://gitlab.com/gitlab-org/gitlab/-/issues/284083
+        ].freeze
+
+        # Called before all on_... have been called
+        # When refining this method, always call `super`
+        def on_new_investigation
+          super
+          track_dynamic_feature_flags!
+          track_usage_data_counters_known_events!
+        end
+
+        def on_casgn(node)
+          _, lhs_name, rhs = *node
+
+          save_used_feature_flag(rhs.value) if lhs_name == :FEATURE_FLAG
+        end
+
+        def on_send(node)
+          return if in_spec?(node)
+          return unless trackable_flag?(node)
+
+          flag_arg = flag_arg(node)
+          flag_value = flag_value(node)
+          return unless flag_value
+
+          if flag_arg_is_str_or_sym?(node)
+            if caller_is_feature_gitaly?(node)
+              save_used_feature_flag("gitaly_#{flag_value}")
+            else
+              save_used_feature_flag(flag_value)
+            end
+
+            if experiment_method?(node) || experimentation_method?(node)
+              # Additionally, mark experiment-related feature flag as used as well
+              matching_feature_flags = defined_feature_flags.select { |flag| flag == "#{flag_value}_experiment_percentage" }
+              matching_feature_flags.each do |matching_feature_flag|
+                puts_if_ci(node, "The '#{matching_feature_flag}' feature flag tracks the #{flag_value} experiment, which is still in use, so we'll mark it as used.")
+                save_used_feature_flag(matching_feature_flag)
+              end
+            end
+          elsif flag_arg_is_send_type?(node)
+            puts_if_ci(node, "Feature flag is dynamic: '#{flag_value}.")
+          elsif flag_arg_is_dstr_or_dsym?(node)
+            str_prefix = flag_arg.children[0]
+            rest_children = flag_arg.children[1..]
+
+            if rest_children.none? { |child| child.str_type? }
+              matching_feature_flags = defined_feature_flags.select { |flag| flag.start_with?(str_prefix.value) }
+              matching_feature_flags.each do |matching_feature_flag|
+                puts_if_ci(node, "The '#{matching_feature_flag}' feature flag starts with '#{str_prefix.value}', so we'll optimistically mark it as used.")
+                save_used_feature_flag(matching_feature_flag)
+              end
+            else
+              puts_if_ci(node, "Interpolated feature flag name has multiple static string parts, we won't track it.")
+            end
+          else
+            puts_if_ci(node, "Feature flag has an unknown type: #{flag_arg.type}.")
+          end
+        end
+
+        private
+
+        def puts_if_ci(node, text)
+          puts "#{text} (call: `#{node.source}`, source: #{node.location.expression.source_buffer.name})" if ENV['CI']
+        end
+
+        def save_used_feature_flag(feature_flag_name)
+          used_feature_flag_file = File.expand_path("../../../tmp/feature_flags/#{feature_flag_name}.used", __dir__)
+          return if File.exist?(used_feature_flag_file)
+
+          FileUtils.touch(used_feature_flag_file)
+        end
+
+        def class_caller(node)
+          node.children[0]&.const_name.to_s
+        end
+
+        def method_name(node)
+          node.children[1]
+        end
+
+        def flag_arg(node)
+          if worker_method?(node)
+            return unless node.children.size > 3
+
+            node.children[3].each_pair.find do |pair|
+              pair.key.value == :feature_flag
+            end&.value
+          elsif graphql_method?(node)
+            return unless node.children.size > 3
+
+            opts_index = node.children[3].hash_type? ? 3 : 4
+            return unless node.children[opts_index]
+
+            node.children[opts_index].each_pair.find do |pair|
+              pair.key.value == :feature_flag
+            end&.value
+          else
+            arg_index = rugged_method?(node) ? 3 : 2
+
+            node.children[arg_index]
+          end
+        end
+
+        def flag_value(node)
+          flag_arg = flag_arg(node)
+          return unless flag_arg
+
+          if flag_arg.respond_to?(:value)
+            flag_arg.value
+          else
+            flag_arg
+          end.to_s.tr("\n/", ' _')
+        end
+
+        def flag_arg_is_str_or_sym?(node)
+          flag_arg = flag_arg(node)
+          flag_arg.str_type? || flag_arg.sym_type?
+        end
+
+        def flag_arg_is_send_type?(node)
+          flag_arg(node).send_type?
+        end
+
+        def flag_arg_is_dstr_or_dsym?(node)
+          flag = flag_arg(node)
+          (flag.dstr_type? || flag.dsym_type?) && flag.children[0].str_type?
+        end
+
+        def caller_is_feature?(node)
+          class_caller(node) == "Feature"
+        end
+
+        def caller_is_feature_gitaly?(node)
+          class_caller(node) == "Feature::Gitaly"
+        end
+
+        def caller_is_experimentation?(node)
+          class_caller(node) == "Gitlab::Experimentation"
+        end
+
+        def experiment_method?(node)
+          EXPERIMENT_METHODS.include?(method_name(node))
+        end
+
+        def rugged_method?(node)
+          RUGGED_METHODS.include?(method_name(node))
+        end
+
+        def feature_method?(node)
+          FEATURE_METHODS.include?(method_name(node)) && (caller_is_feature?(node) || caller_is_feature_gitaly?(node))
+        end
+
+        def experimentation_method?(node)
+          EXPERIMENTATION_METHODS.include?(method_name(node)) && caller_is_experimentation?(node)
+        end
+
+        def worker_method?(node)
+          WORKER_METHODS.include?(method_name(node))
+        end
+
+        def graphql_method?(node)
+          GRAPHQL_METHODS.include?(method_name(node)) && in_graphql_types?(node)
+        end
+
+        def self_method?(node)
+          SELF_METHODS.include?(method_name(node)) && class_caller(node).empty?
+        end
+
+        def trackable_flag?(node)
+          feature_method?(node) || experimentation_method?(node) || graphql_method?(node) || self_method?(node)
+        end
+
+        # Marking all event's feature flags as used as Gitlab::UsageDataCounters::HLLRedisCounter.track_event{,context}
+        # is mostly used with dynamic event name.
+        def track_dynamic_feature_flags!
+          DYNAMIC_FEATURE_FLAGS.each(&method(:save_used_feature_flag))
+        end
+
+        # Marking all event's feature flags as used as Gitlab::UsageDataCounters::HLLRedisCounter.track_event{,context}
+        # is mostly used with dynamic event name.
+        def track_usage_data_counters_known_events!
+          usage_data_counters_known_event_feature_flags.each(&method(:save_used_feature_flag))
+        end
+
+        def usage_data_counters_known_event_feature_flags
+          USAGE_DATA_COUNTERS_EVENTS_YAML_GLOBS.each_with_object(Set.new) do |glob, memo|
+            Dir.glob(glob).each do |path|
+              YAML.safe_load(File.read(path))&.each do |hash|
+                memo << hash['feature_flag'] if hash['feature_flag']
+              end
+            end
+          end
+        end
+
+        def defined_feature_flags
+          @defined_feature_flags ||= begin
+            flags_paths = [
+              'config/feature_flags/**/*.yml'
+            ]
+
+            # For EE additionally process `ee/` feature flags
+            if File.exist?(File.expand_path('../../../ee/app/models/license.rb', __dir__)) && !%w[true 1].include?(ENV['FOSS_ONLY'].to_s)
+              flags_paths << 'ee/config/feature_flags/**/*.yml'
+            end
+
+            flags_paths.each_with_object([]) do |flags_path, memo|
+              flags_path = File.expand_path("../../../#{flags_path}", __dir__)
+              Dir.glob(flags_path).each do |path|
+                feature_flag_name = File.basename(path, '.yml')
+
+                memo << feature_flag_name
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/rubocop/cop/migration/prevent_index_creation.rb b/rubocop/cop/migration/prevent_index_creation.rb
new file mode 100644
index 00000000000..c90f911d24e
--- /dev/null
+++ b/rubocop/cop/migration/prevent_index_creation.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+require_relative '../../migration_helpers'
+
+module RuboCop
+  module Cop
+    module Migration
+      # Cop that checks if new indexes are introduced to forbidden tables.
+      class PreventIndexCreation < RuboCop::Cop::Cop
+        include MigrationHelpers
+
+        FORBIDDEN_TABLES = %i[ci_builds].freeze
+
+        MSG = "Adding new index to #{FORBIDDEN_TABLES.join(", ")} is forbidden, see https://gitlab.com/gitlab-org/gitlab/-/issues/332886"
+
+        def_node_matcher :add_index?, <<~PATTERN
+          (send nil? :add_index (sym #forbidden_tables?) ...)
+        PATTERN
+
+        def_node_matcher :add_concurrent_index?, <<~PATTERN
+          (send nil? :add_concurrent_index (sym #forbidden_tables?) ...)
+        PATTERN
+
+        def forbidden_tables?(node)
+          FORBIDDEN_TABLES.include?(node)
+        end
+
+        def on_def(node)
+          return unless in_migration?(node)
+
+          node.each_descendant(:send) do |send_node|
+            add_offense(send_node, location: :selector) if offense?(send_node)
+          end
+        end
+
+        def offense?(node)
+          add_index?(node) || add_concurrent_index?(node)
+        end
+      end
+    end
+  end
+end
diff --git a/rubocop/cop/migration/sidekiq_queue_migrate.rb b/rubocop/cop/migration/sidekiq_queue_migrate.rb
new file mode 100644
index 00000000000..134bda590da
--- /dev/null
+++ b/rubocop/cop/migration/sidekiq_queue_migrate.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative '../../migration_helpers'
+
+module RuboCop
+  module Cop
+    module Migration
+      # Cop that checks if sidekiq_queue_migrate is used in a regular
+      # (not post-deployment) migration.
+      class SidekiqQueueMigrate < RuboCop::Cop::Cop
+        include MigrationHelpers
+
+        MSG = '`sidekiq_queue_migrate` must only be used in post-deployment migrations'
+
+        def on_def(node)
+          return unless in_migration?(node) && !in_post_deployment_migration?(node)
+
+          node.each_descendant(:send) do |send_node|
+            send_method = send_node.children[1]
+
+            if send_method == :sidekiq_queue_migrate
+              add_offense(send_node, location: :selector)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/rubocop/cop/migration/with_lock_retries_disallowed_method.rb b/rubocop/cop/migration/with_lock_retries_disallowed_method.rb
index cb36e7413ab..b3d05ad1a6d 100644
--- a/rubocop/cop/migration/with_lock_retries_disallowed_method.rb
+++ b/rubocop/cop/migration/with_lock_retries_disallowed_method.rb
@@ -22,6 +22,7 @@ module RuboCop
           remove_foreign_key_if_exists
           remove_foreign_key_without_error
           rename_index
+          rename_constraint
           table_exists?
           index_exists_by_name?
           foreign_key_exists?
diff --git a/rubocop/cop/worker_data_consistency.rb b/rubocop/cop/worker_data_consistency.rb
new file mode 100644
index 00000000000..e9047750f3e
--- /dev/null
+++ b/rubocop/cop/worker_data_consistency.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative '../code_reuse_helpers'
+
+module RuboCop
+  module Cop
+    # This cop checks for a call to `data_consistency` to exist in Sidekiq workers.
+    #
+    # @example
+    #
+    # # bad
+    # class BadWorker
+    #   def perform
+    #   end
+    # end
+    #
+    # # good
+    # class GoodWorker
+    #   data_consistency :delayed
+    #
+    #   def perform
+    #   end
+    # end
+    #
+    class WorkerDataConsistency < RuboCop::Cop::Cop
+      include CodeReuseHelpers
+
+      HELP_LINK = 'https://docs.gitlab.com/ee/development/sidekiq_style_guide.html#job-data-consistency-strategies'
+
+      MSG = <<~MSG
+        Should define data_consistency expectation.
+
+        It is encouraged for workers to use database replicas as much as possible by declaring
+        data_consistency to use the :delayed or :sticky modes. Mode :always will result in the
+        worker always hitting the primary database node for both reads and writes, which limits
+        scalability.
+
+        Some guidelines:
+        
+        1. If your worker mostly writes or reads its own writes, use mode :always. TRY TO AVOID THIS.
+        2. If your worker performs mostly reads and can tolerate small delays, use mode :delayed.
+        3. If your worker performs mostly reads but cannot tolerate any delays, use mode :sticky.
+
+        See #{HELP_LINK} for a more detailed explanation of these settings.
+      MSG
+
+      def_node_search :application_worker?, <<~PATTERN
+      `(send nil? :include (const nil? :ApplicationWorker))
+      PATTERN
+
+      def_node_search :data_consistency_defined?, <<~PATTERN
+        `(send nil? :data_consistency ...)
+      PATTERN
+
+      def on_class(node)
+        return unless in_worker?(node) && application_worker?(node)
+        return if data_consistency_defined?(node)
+
+        add_offense(node, location: :expression)
+      end
+    end
+  end
+end
diff --git a/rubocop/rubocop-migrations.yml b/rubocop/rubocop-migrations.yml
index c8f50016710..979b0fa2936 100644
--- a/rubocop/rubocop-migrations.yml
+++ b/rubocop/rubocop-migrations.yml
@@ -42,6 +42,7 @@ Migration/UpdateLargeTable:
     - :user_details
     - :vulnerability_occurrences
     - :web_hook_logs
+    - :vulnerabilities
   DeniedMethods:
     - :change_column_type_concurrently
     - :rename_column_concurrently