Add latest changes from gitlab-org/gitlab@13-5-stable-eev13.5.0-rc42

18 files changed, 154 insertions, 82 deletions

diff --git a/app/models/concerns/approvable_base.rb b/app/models/concerns/approvable_base.rb
index d07c4ec43ac..c2d94b50f8d 100644
--- a/app/models/concerns/approvable_base.rb
+++ b/app/models/concerns/approvable_base.rb
@@ -2,10 +2,34 @@
 
 module ApprovableBase
   extend ActiveSupport::Concern
+  include FromUnion
 
   included do
     has_many :approvals, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent
     has_many :approved_by_users, through: :approvals, source: :user
+
+    scope :without_approvals, -> { left_outer_joins(:approvals).where(approvals: { id: nil }) }
+    scope :with_approvals, -> { joins(:approvals) }
+    scope :approved_by_users_with_ids, -> (*user_ids) do
+      with_approvals
+        .merge(Approval.with_user)
+        .where(users: { id: user_ids })
+        .group(:id)
+        .having("COUNT(users.id) = ?", user_ids.size)
+    end
+    scope :approved_by_users_with_usernames, -> (*usernames) do
+      with_approvals
+        .merge(Approval.with_user)
+        .where(users: { username: usernames })
+        .group(:id)
+        .having("COUNT(users.id) = ?", usernames.size)
+    end
+  end
+
+  class_methods do
+    def select_from_union(relations)
+      where(id: from_union(relations))
+    end
   end
 
   def approved_by?(user)
diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb
index 0dd55ab67b5..d342b526677 100644
--- a/app/models/concerns/avatarable.rb
+++ b/app/models/concerns/avatarable.rb
@@ -3,16 +3,11 @@
 module Avatarable
   extend ActiveSupport::Concern
 
-  ALLOWED_IMAGE_SCALER_WIDTHS = [
-    400,
-    200,
-    64,
-    48,
-    40,
-    26,
-    20,
-    16
-  ].freeze
+  USER_AVATAR_SIZES = [16, 20, 23, 24, 26, 32, 36, 38, 40, 48, 60, 64, 90, 96, 120, 160].freeze
+  PROJECT_AVATAR_SIZES = [15, 40, 48, 64, 88].freeze
+  GROUP_AVATAR_SIZES = [15, 37, 38, 39, 40, 64, 96].freeze
+
+  ALLOWED_IMAGE_SCALER_WIDTHS = (USER_AVATAR_SIZES | PROJECT_AVATAR_SIZES | GROUP_AVATAR_SIZES).freeze
 
   included do
     prepend ShadowMethods
diff --git a/app/models/concerns/checksummable.rb b/app/models/concerns/checksummable.rb
index d6d17bfc604..056abafd0ce 100644
--- a/app/models/concerns/checksummable.rb
+++ b/app/models/concerns/checksummable.rb
@@ -3,11 +3,11 @@
 module Checksummable
   extend ActiveSupport::Concern
 
-  def crc32(data)
-    Zlib.crc32(data)
-  end
-
   class_methods do
+    def crc32(data)
+      Zlib.crc32(data)
+    end
+
     def hexdigest(path)
       ::Digest::SHA256.file(path).hexdigest
     end
diff --git a/app/models/concerns/counter_attribute.rb b/app/models/concerns/counter_attribute.rb
index a5c7393e8f7..b468415c4c7 100644
--- a/app/models/concerns/counter_attribute.rb
+++ b/app/models/concerns/counter_attribute.rb
@@ -20,6 +20,14 @@
 # To increment the counter we can use the method:
 #   delayed_increment_counter(:commit_count, 3)
 #
+# It is possible to register callbacks to be executed after increments have
+# been flushed to the database. Callbacks are not executed if there are no increments
+# to flush.
+#
+#  counter_attribute_after_flush do |statistic|
+#    Namespaces::ScheduleAggregationWorker.perform_async(statistic.namespace_id)
+#  end
+#
 module CounterAttribute
   extend ActiveSupport::Concern
   extend AfterCommitQueue
@@ -48,6 +56,15 @@ module CounterAttribute
     def counter_attributes
       @counter_attributes ||= Set.new
     end
+
+    def after_flush_callbacks
+      @after_flush_callbacks ||= []
+    end
+
+    # perform registered callbacks after increments have been flushed to the database
+    def counter_attribute_after_flush(&callback)
+      after_flush_callbacks << callback
+    end
   end
 
   # This method must only be called by FlushCounterIncrementsWorker
@@ -75,6 +92,8 @@ module CounterAttribute
         unsafe_update_counters(id, attribute => increment_value)
         redis_state { |redis| redis.del(flushed_key) }
       end
+
+      execute_after_flush_callbacks
     end
   end
 
@@ -108,13 +127,13 @@ module CounterAttribute
     counter_key(attribute) + ':lock'
   end
 
-  private
-
   def counter_attribute_enabled?(attribute)
     Feature.enabled?(:efficient_counter_attribute, project) &&
       self.class.counter_attributes.include?(attribute)
   end
 
+  private
+
   def steal_increments(increment_key, flushed_key)
     redis_state do |redis|
       redis.eval(LUA_STEAL_INCREMENT_SCRIPT, keys: [increment_key, flushed_key])
@@ -129,6 +148,12 @@ module CounterAttribute
     self.class.update_counters(id, increments)
   end
 
+  def execute_after_flush_callbacks
+    self.class.after_flush_callbacks.each do |callback|
+      callback.call(self)
+    end
+  end
+
   def redis_state(&block)
     Gitlab::Redis::SharedState.with(&block)
   end
diff --git a/app/models/concerns/has_repository.rb b/app/models/concerns/has_repository.rb
index d909b67d7ba..978a54bdee7 100644
--- a/app/models/concerns/has_repository.rb
+++ b/app/models/concerns/has_repository.rb
@@ -71,6 +71,10 @@ module HasRepository
     raise NotImplementedError
   end
 
+  def lfs_enabled?
+    false
+  end
+
   def empty_repo?
     repository.empty?
   end
@@ -80,7 +84,11 @@ module HasRepository
   end
 
   def default_branch_from_preferences
-    empty_repo? ? Gitlab::CurrentSettings.default_branch_name : nil
+    return unless empty_repo?
+
+    group_branch_default_name = group&.default_branch_name if respond_to?(:group)
+
+    group_branch_default_name || Gitlab::CurrentSettings.default_branch_name
   end
 
   def reload_default_branch
diff --git a/app/models/concerns/has_user_type.rb b/app/models/concerns/has_user_type.rb
index 8a238dc736c..468387115e5 100644
--- a/app/models/concerns/has_user_type.rb
+++ b/app/models/concerns/has_user_type.rb
@@ -11,16 +11,18 @@ module HasUserType
     service_user: 4,
     ghost: 5,
     project_bot: 6,
-    migration_bot: 7
+    migration_bot: 7,
+    security_bot: 8
   }.with_indifferent_access.freeze
 
-  BOT_USER_TYPES = %w[alert_bot project_bot support_bot visual_review_bot migration_bot].freeze
+  BOT_USER_TYPES = %w[alert_bot project_bot support_bot visual_review_bot migration_bot security_bot].freeze
   NON_INTERNAL_USER_TYPES = %w[human project_bot service_user].freeze
   INTERNAL_USER_TYPES = (USER_TYPES.keys - NON_INTERNAL_USER_TYPES).freeze
 
   included do
     scope :humans, -> { where(user_type: :human) }
     scope :bots, -> { where(user_type: BOT_USER_TYPES) }
+    scope :without_bots, -> { humans.or(where.not(user_type: BOT_USER_TYPES)) }
     scope :bots_without_project_bot, -> { where(user_type: BOT_USER_TYPES - ['project_bot']) }
     scope :non_internal, -> { humans.or(where(user_type: NON_INTERNAL_USER_TYPES)) }
     scope :without_ghosts, -> { humans.or(where.not(user_type: :ghost)) }
diff --git a/app/models/concerns/integration.rb b/app/models/concerns/integration.rb
index 34ff5bb1195..9d446841a9f 100644
--- a/app/models/concerns/integration.rb
+++ b/app/models/concerns/integration.rb
@@ -16,7 +16,7 @@ module Integration
       Project.where(id: custom_integration_project_ids)
     end
 
-    def ids_without_integration(integration, limit)
+    def without_integration(integration)
       services = Service
         .select('1')
         .where('services.project_id = projects.id')
@@ -26,8 +26,6 @@ module Integration
         .where('NOT EXISTS (?)', services)
         .where(pending_delete: false)
         .where(archived: false)
-        .limit(limit)
-        .pluck(:id)
     end
   end
 end
diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb
index 888e1b384a2..7624a1a4e80 100644
--- a/app/models/concerns/issuable.rb
+++ b/app/models/concerns/issuable.rb
@@ -182,7 +182,7 @@ module Issuable
     end
 
     def supports_time_tracking?
-      is_a?(TimeTrackable) && !incident?
+      is_a?(TimeTrackable)
     end
 
     def supports_severity?
@@ -203,15 +203,6 @@ module Issuable
       issuable_severity&.severity || IssuableSeverity::DEFAULT
     end
 
-    def update_severity(severity)
-      return unless incident?
-
-      severity = severity.to_s.downcase
-      severity = IssuableSeverity::DEFAULT unless IssuableSeverity.severities.key?(severity)
-
-      (issuable_severity || build_issuable_severity(issue_id: id)).update(severity: severity)
-    end
-
     private
 
     def description_max_length_for_new_records_is_valid
diff --git a/app/models/concerns/issue_available_features.rb b/app/models/concerns/issue_available_features.rb
new file mode 100644
index 00000000000..6efb8103b7b
--- /dev/null
+++ b/app/models/concerns/issue_available_features.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Verifies features availability based on issue type.
+# This can be used, for example, for hiding UI elements or blocking specific
+# quick actions for particular issue types;
+module IssueAvailableFeatures
+  extend ActiveSupport::Concern
+
+  # EE only features are listed on EE::IssueAvailableFeatures
+  def available_features_for_issue_types
+    {}.with_indifferent_access
+  end
+
+  def issue_type_supports?(feature)
+    unless available_features_for_issue_types.has_key?(feature)
+      raise ArgumentError, 'invalid feature'
+    end
+
+    available_features_for_issue_types[feature].include?(issue_type)
+  end
+end
+
+IssueAvailableFeatures.prepend_if_ee('EE::IssueAvailableFeatures')
diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb
index 7b4485376d4..b10e8547e86 100644
--- a/app/models/concerns/mentionable.rb
+++ b/app/models/concerns/mentionable.rb
@@ -81,13 +81,6 @@ module Mentionable
   end
 
   def store_mentions!
-    # if store_mentioned_users_to_db feature flag is not enabled then consider storing operation as succeeded
-    # because we wrap this method in transaction with with_transaction_returning_status, and we need the status to be
-    # successful if mentionable.save is successful.
-    #
-    # This line will get removed when we remove the feature flag.
-    return true unless store_mentioned_users_to_db_enabled?
-
     refs = all_references(self.author)
 
     references = {}
@@ -253,15 +246,6 @@ module Mentionable
   def model_user_mention
     user_mentions.where(note_id: nil).first_or_initialize
   end
-
-  # We need this method to be checking that store_mentioned_users_to_db feature flag is enabled at the group level
-  # and not the project level as epics are defined at group level and we want to have epics store user mentions as well
-  # for the test period.
-  # During the test period the flag should be enabled at the group level.
-  def store_mentioned_users_to_db_enabled?
-    return Feature.enabled?(:store_mentioned_users_to_db, self.project&.group, default_enabled: true) if self.respond_to?(:project)
-    return Feature.enabled?(:store_mentioned_users_to_db, self.group, default_enabled: true) if self.respond_to?(:group)
-  end
 end
 
 Mentionable.prepend_if_ee('EE::Mentionable')
diff --git a/app/models/concerns/presentable.rb b/app/models/concerns/presentable.rb
index 06c300c2e41..1f05abff2f4 100644
--- a/app/models/concerns/presentable.rb
+++ b/app/models/concerns/presentable.rb
@@ -5,13 +5,13 @@ module Presentable
 
   class_methods do
     def present(attributes)
-      all.map { |klass_object| klass_object.present(attributes) }
+      all.map { |klass_object| klass_object.present(**attributes) }
     end
   end
 
   def present(**attributes)
     Gitlab::View::Presenter::Factory
-      .new(self, attributes)
+      .new(self, **attributes)
       .fabricate!
   end
 end
diff --git a/app/models/concerns/reactive_caching.rb b/app/models/concerns/reactive_caching.rb
index 5f30fc0c36c..3470bdab5fb 100644
--- a/app/models/concerns/reactive_caching.rb
+++ b/app/models/concerns/reactive_caching.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # The usage of the ReactiveCaching module is documented here:
-# https://docs.gitlab.com/ee/development/utilities.html#reactivecaching
+# https://docs.gitlab.com/ee/development/reactive_caching.md
 module ReactiveCaching
   extend ActiveSupport::Concern
 
@@ -9,7 +9,7 @@ module ReactiveCaching
   ExceededReactiveCacheLimit = Class.new(StandardError)
 
   WORK_TYPE = {
-    default: ReactiveCachingWorker,
+    no_dependency: ReactiveCachingWorker,
     external_dependency: ExternalServiceReactiveCachingWorker
   }.freeze
 
@@ -30,7 +30,6 @@ module ReactiveCaching
     self.reactive_cache_refresh_interval = 1.minute
     self.reactive_cache_lifetime = 10.minutes
     self.reactive_cache_hard_limit = nil # this value should be set in megabytes. E.g: 1.megabyte
-    self.reactive_cache_work_type = :default
     self.reactive_cache_worker_finder = ->(id, *_args) do
       find_by(primary_key => id)
     end
diff --git a/app/models/concerns/reactive_service.rb b/app/models/concerns/reactive_service.rb
index af69da24994..c444f238944 100644
--- a/app/models/concerns/reactive_service.rb
+++ b/app/models/concerns/reactive_service.rb
@@ -8,5 +8,6 @@ module ReactiveService
 
     # Default cache key: class name + project_id
     self.reactive_cache_key = ->(service) { [service.class.model_name.singular, service.project_id] }
+    self.reactive_cache_work_type = :external_dependency
   end
 end
diff --git a/app/models/concerns/referable.rb b/app/models/concerns/referable.rb
index 40edd3b3ead..9a17131c91c 100644
--- a/app/models/concerns/referable.rb
+++ b/app/models/concerns/referable.rb
@@ -85,7 +85,7 @@ module Referable
           \/#{route.is_a?(Regexp) ? route : Regexp.escape(route)}
           \/#{pattern}
           (?<path>
-            (\/[a-z0-9_=-]+)*
+            (\/[a-z0-9_=-]+)*\/*
           )?
           (?<query>
             \?[a-z0-9_=-]+
diff --git a/app/models/concerns/relative_positioning.rb b/app/models/concerns/relative_positioning.rb
index 3cbc174536c..7f559f0a7ed 100644
--- a/app/models/concerns/relative_positioning.rb
+++ b/app/models/concerns/relative_positioning.rb
@@ -102,33 +102,16 @@ module RelativePositioning
       delta = at_end ? gap : -gap
       indexed = (at_end ? objects : objects.reverse).each_with_index
 
-      # Some classes are polymorphic, and not all siblings are in the same table.
-      by_model = indexed.group_by { |pair| pair.first.class }
       lower_bound, upper_bound = at_end ? [position, MAX_POSITION] : [MIN_POSITION, position]
 
-      by_model.each do |model, pairs|
-        model.transaction do
-          pairs.each_slice(100) do |batch|
-            # These are known to be integers, one from the DB, and the other
-            # calculated by us, and thus safe to interpolate
-            values = batch.map do |obj, i|
-              desired_pos = position + delta * (i + 1)
-              pos = desired_pos.clamp(lower_bound, upper_bound)
-              obj.relative_position = pos
-              "(#{obj.id}, #{pos})"
-            end.join(', ')
-
-            model.connection.exec_query(<<~SQL, "UPDATE #{model.table_name} positions")
-              WITH cte(cte_id, new_pos) AS (
-               SELECT *
-               FROM (VALUES #{values}) as t (id, pos)
-              )
-              UPDATE #{model.table_name}
-              SET relative_position = cte.new_pos
-              FROM cte
-              WHERE cte_id = id
-            SQL
+      representative.model_class.transaction do
+        indexed.each_slice(100) do |batch|
+          mapping = batch.to_h.transform_values! do |i|
+            desired_pos = position + delta * (i + 1)
+            { relative_position: desired_pos.clamp(lower_bound, upper_bound) }
           end
+
+          ::Gitlab::Database::BulkUpdate.execute([:relative_position], mapping, &:model_class)
         end
       end
 
@@ -200,4 +183,16 @@ module RelativePositioning
   # Override if you want to be notified of failures to move
   def could_not_move(exception)
   end
+
+  # Override if the implementing class is not a simple application record, for
+  # example if the record is loaded from a union.
+  def reset_relative_position
+    reset.relative_position
+  end
+
+  # Override if the model class needs a more complicated computation (e.g. the
+  # object is a member of a union).
+  def model_class
+    self.class
+  end
 end
diff --git a/app/models/concerns/shardable.rb b/app/models/concerns/shardable.rb
index 57cd77b44b4..c0883c08289 100644
--- a/app/models/concerns/shardable.rb
+++ b/app/models/concerns/shardable.rb
@@ -5,6 +5,10 @@ module Shardable
 
   included do
     belongs_to :shard
+
+    scope :for_repository_storage, -> (repository_storage) { joins(:shard).where(shards: { name: repository_storage }) }
+    scope :excluding_repository_storage, -> (repository_storage) { joins(:shard).where.not(shards: { name: repository_storage }) }
+
     validates :shard, presence: true
   end
 
diff --git a/app/models/concerns/timebox.rb b/app/models/concerns/timebox.rb
index 3e2cf9031d0..23fd73f2904 100644
--- a/app/models/concerns/timebox.rb
+++ b/app/models/concerns/timebox.rb
@@ -73,6 +73,32 @@ module Timebox
       end
     end
 
+    # A timebox is within the timeframe (start_date, end_date) if it overlaps
+    # with that timeframe:
+    #
+    #        [  timeframe   ]
+    #  ----| ................     # Not overlapping
+    #   |--| ................     # Not overlapping
+    #  ------|...............     # Overlapping
+    #  -----------------------|   # Overlapping
+    #  ---------|............     # Overlapping
+    #     |-----|............     # Overlapping
+    #        |--------------|     # Overlapping
+    #     |--------------------|  # Overlapping
+    #        ...|-----|......     # Overlapping
+    #        .........|-----|     # Overlapping
+    #        .........|---------  # Overlapping
+    #      |--------------------  # Overlapping
+    #        .........|--------|  # Overlapping
+    #        ...............|--|  # Overlapping
+    #        ............... |-|  # Not Overlapping
+    #        ............... |--  # Not Overlapping
+    #
+    # where: . = in timeframe
+    #        ---| no start
+    #        |--- no end
+    #        |--| defined start and end
+    #
     scope :within_timeframe, -> (start_date, end_date) do
       where('start_date is not NULL or due_date is not NULL')
         .where('start_date is NULL or start_date <= ?', end_date)
diff --git a/app/models/concerns/update_project_statistics.rb b/app/models/concerns/update_project_statistics.rb
index a7028e18451..586f1dbb65c 100644
--- a/app/models/concerns/update_project_statistics.rb
+++ b/app/models/concerns/update_project_statistics.rb
@@ -80,10 +80,7 @@ module UpdateProjectStatistics
 
       run_after_commit do
         ProjectStatistics.increment_statistic(
-          project_id, self.class.project_statistics_name, delta)
-
-        Namespaces::ScheduleAggregationWorker.perform_async(
-          project.namespace_id)
+          project, self.class.project_statistics_name, delta)
       end
     end
   end