Merge remote-tracking branch 'upstream/master' into 14995-custom_wiki_sidebar

* upstream/master: (4180 commits)
  Enable frozen string literals for app/workers/*.rb
  Resolve "Search dropdown hides & shows when typing"
  Revert merge request widget button max height
  Update CHANGELOG.md for 11.0.2
  Update external link icon in header user dropdown
  Added Diff Viewer to new VUE based MR page
  Fixed eslint failure in IDE spec helpers
  Use refs instead of querySelector.
  Show file in tree on WebIDE open
  Resolve "Remove unused bootstrap component CSS"
  Resolve "Explain what Groups are in the New Group page"
  [QA] Make sure we wait for the deploy key list to load
  Update _scopes_form.html.haml to remove duplicate information
  Use the branch instead of the tag to install
  port the EE changes
  Add index on deployable_type/id for deployments
  Add a helper to rename a column using a background migration
  Fix performance bottleneck when rendering large wiki pages
  Port Namespace#root_ancestor to CE
  Remove duplicate spec
  ...

33 files changed, 578 insertions, 107 deletions

diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb
index 4b66725a3e6..164c704260e 100644
--- a/app/models/concerns/atomic_internal_id.rb
+++ b/app/models/concerns/atomic_internal_id.rb
@@ -25,22 +25,23 @@ module AtomicInternalId
   extend ActiveSupport::Concern
 
   module ClassMethods
-    def has_internal_id(column, scope:, init:) # rubocop:disable Naming/PredicateName
-      before_validation(on: :create) do
-        if read_attribute(column).blank?
-          scope_attrs = { scope => association(scope).reader }
+    def has_internal_id(column, scope:, init:, presence: true) # rubocop:disable Naming/PredicateName
+      before_validation :"ensure_#{scope}_#{column}!", on: :create
+      validates column, presence: presence
+
+      define_method("ensure_#{scope}_#{column}!") do
+        scope_value = association(scope).reader
+
+        if read_attribute(column).blank? && scope_value
+          scope_attrs = { scope_value.class.table_name.singularize.to_sym => scope_value }
           usage = self.class.table_name.to_sym
 
           new_iid = InternalId.generate_next(self, scope_attrs, usage, init)
           write_attribute(column, new_iid)
         end
-      end
 
-      validates column, presence: true, numericality: true
+        read_attribute(column)
+      end
     end
   end
-
-  def to_param
-    iid.to_s
-  end
 end
diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb
index d35e37935fb..095897b08e3 100644
--- a/app/models/concerns/avatarable.rb
+++ b/app/models/concerns/avatarable.rb
@@ -3,11 +3,15 @@ module Avatarable
 
   included do
     prepend ShadowMethods
+    include ObjectStorage::BackgroundMove
+    include Gitlab::Utils::StrongMemoize
 
     validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? }
     validates :avatar, file_size: { maximum: 200.kilobytes.to_i }
 
     mount_uploader :avatar, AvatarUploader
+
+    after_initialize :add_avatar_to_batch
   end
 
   module ShadowMethods
@@ -17,11 +21,22 @@ module Avatarable
 
       avatar_path(only_path: args.fetch(:only_path, true)) || super
     end
+
+    def retrieve_upload(identifier, paths)
+      upload = retrieve_upload_from_batch(identifier)
+
+      # This fallback is needed when deleting an upload, because we may have
+      # already been removed from the DB. We have to check an explicit `#nil?`
+      # because it's a BatchLoader instance.
+      upload = super if upload.nil?
+
+      upload
+    end
   end
 
   def avatar_type
     unless self.avatar.image?
-      self.errors.add :avatar, "only images allowed"
+      errors.add :avatar, "file format is not supported. Please try one of the following supported formats: #{AvatarUploader::IMAGE_EXT.join(', ')}"
     end
   end
 
@@ -30,12 +45,13 @@ module Avatarable
 
     asset_host = ActionController::Base.asset_host
     use_asset_host = asset_host.present?
+    use_authentication = respond_to?(:public?) && !public?
 
     # Avatars for private and internal groups and projects require authentication to be viewed,
     # which means they can only be served by Rails, on the regular GitLab host.
     # If an asset host is configured, we need to return the fully qualified URL
     # instead of only the avatar path, so that Rails doesn't prefix it with the asset host.
-    if use_asset_host && respond_to?(:public?) && !public?
+    if use_asset_host && use_authentication
       use_asset_host = false
       only_path = false
     end
@@ -48,6 +64,39 @@ module Avatarable
       url_base << gitlab_config.relative_url_root
     end
 
-    url_base + avatar.url
+    url_base + avatar.local_url
+  end
+
+  # Path that is persisted in the tracking Upload model. Used to fetch the
+  # upload from the model.
+  def upload_paths(identifier)
+    avatar_mounter.blank_uploader.store_dirs.map { |store, path| File.join(path, identifier) }
+  end
+
+  private
+
+  def retrieve_upload_from_batch(identifier)
+    BatchLoader.for(identifier: identifier, model: self).batch(key: self.class) do |upload_params, loader, args|
+      model_class = args[:key]
+      paths = upload_params.flat_map do |params|
+        params[:model].upload_paths(params[:identifier])
+      end
+
+      Upload.where(uploader: AvatarUploader, path: paths).find_each do |upload|
+        model = model_class.instantiate('id' => upload.model_id)
+
+        loader.call({ model: model, identifier: File.basename(upload.path) }, upload)
+      end
+    end
+  end
+
+  def add_avatar_to_batch
+    return unless avatar_mounter
+
+    avatar_mounter.read_identifiers.each { |identifier| retrieve_upload_from_batch(identifier) }
+  end
+
+  def avatar_mounter
+    strong_memoize(:avatar_mounter) { _mounter(:avatar) }
   end
 end
diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb
index d8394415362..fce37e7f78e 100644
--- a/app/models/concerns/awardable.rb
+++ b/app/models/concerns/awardable.rb
@@ -79,11 +79,7 @@ module Awardable
   end
 
   def user_can_award?(current_user, name)
-    if user_authored?(current_user)
-      !awardable_votes?(normalize_name(name))
-    else
-      true
-    end
+    awardable_by_user?(current_user, name) && Ability.allowed?(current_user, :award_emoji, self)
   end
 
   def user_authored?(current_user)
@@ -119,4 +115,12 @@ module Awardable
   def normalize_name(name)
     Gitlab::Emoji.normalize_emoji_name(name)
   end
+
+  def awardable_by_user?(current_user, name)
+    if user_authored?(current_user)
+      !awardable_votes?(normalize_name(name))
+    else
+      true
+    end
+  end
 end
diff --git a/app/models/concerns/batch_destroy_dependent_associations.rb b/app/models/concerns/batch_destroy_dependent_associations.rb
new file mode 100644
index 00000000000..353ee2e73d0
--- /dev/null
+++ b/app/models/concerns/batch_destroy_dependent_associations.rb
@@ -0,0 +1,28 @@
+# Provides a way to work around Rails issue where dependent objects are all
+# loaded into memory before destroyed: https://github.com/rails/rails/issues/22510.
+#
+# This concern allows an ActiveRecord module to destroy all its dependent
+# associations in batches. The idea is borrowed from https://github.com/thisismydesign/batch_dependent_associations.
+#
+# The differences here with that gem:
+#
+# 1. We allow excluding certain associations.
+# 2. We don't need to support delete_all since we can use the EachBatch concern.
+module BatchDestroyDependentAssociations
+  extend ActiveSupport::Concern
+
+  DEPENDENT_ASSOCIATIONS_BATCH_SIZE = 1000
+
+  def dependent_associations_to_destroy
+    self.class.reflect_on_all_associations(:has_many).select { |assoc| assoc.options[:dependent] == :destroy }
+  end
+
+  def destroy_dependent_associations_in_batches(exclude: [])
+    dependent_associations_to_destroy.each do |association|
+      next if exclude.include?(association.name)
+
+      # rubocop:disable GitlabSecurity/PublicSend
+      public_send(association.name).find_each(batch_size: DEPENDENT_ASSOCIATIONS_BATCH_SIZE, &:destroy)
+    end
+  end
+end
diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb
index 4ae5dd8c677..9f6358cecbe 100644
--- a/app/models/concerns/cache_markdown_field.rb
+++ b/app/models/concerns/cache_markdown_field.rb
@@ -11,7 +11,9 @@ module CacheMarkdownField
   extend ActiveSupport::Concern
 
   # Increment this number every time the renderer changes its output
-  CACHE_VERSION = 3
+  CACHE_REDCARPET_VERSION         = 3
+  CACHE_COMMONMARK_VERSION_START  = 10
+  CACHE_COMMONMARK_VERSION        = 11
 
   # changes to these attributes cause the cache to be invalidates
   INVALIDATED_BY = %w[author project].freeze
@@ -49,12 +51,14 @@ module CacheMarkdownField
 
     # Always include a project key, or Banzai complains
     project = self.project if self.respond_to?(:project)
-    group = self.group if self.respond_to?(:group)
+    group   = self.group if self.respond_to?(:group)
     context = cached_markdown_fields[field].merge(project: project, group: group)
 
     # Banzai is less strict about authors, so don't always have an author key
     context[:author] = self.author if self.respond_to?(:author)
 
+    context[:markdown_engine] = markdown_engine
+
     context
   end
 
@@ -69,7 +73,7 @@ module CacheMarkdownField
         Banzai::Renderer.cacheless_render_field(self, markdown_field, options)
       ]
     end.to_h
-    updates['cached_markdown_version'] = CacheMarkdownField::CACHE_VERSION
+    updates['cached_markdown_version'] = latest_cached_markdown_version
 
     updates.each {|html_field, data| write_attribute(html_field, data) }
   end
@@ -90,7 +94,7 @@ module CacheMarkdownField
     markdown_changed = attribute_changed?(markdown_field) || false
     html_changed = attribute_changed?(html_field) || false
 
-    CacheMarkdownField::CACHE_VERSION == cached_markdown_version &&
+    latest_cached_markdown_version == cached_markdown_version &&
       (html_changed || markdown_changed == html_changed)
   end
 
@@ -109,6 +113,24 @@ module CacheMarkdownField
     __send__(cached_markdown_fields.html_field(markdown_field)) # rubocop:disable GitlabSecurity/PublicSend
   end
 
+  def latest_cached_markdown_version
+    return CacheMarkdownField::CACHE_COMMONMARK_VERSION unless cached_markdown_version
+
+    if cached_markdown_version < CacheMarkdownField::CACHE_COMMONMARK_VERSION_START
+      CacheMarkdownField::CACHE_REDCARPET_VERSION
+    else
+      CacheMarkdownField::CACHE_COMMONMARK_VERSION
+    end
+  end
+
+  def markdown_engine
+    if latest_cached_markdown_version < CacheMarkdownField::CACHE_COMMONMARK_VERSION_START
+      :redcarpet
+    else
+      :common_mark
+    end
+  end
+
   included do
     cattr_reader :cached_markdown_fields do
       FieldData.new
diff --git a/app/models/concerns/cacheable_attributes.rb b/app/models/concerns/cacheable_attributes.rb
new file mode 100644
index 00000000000..d58d7165969
--- /dev/null
+++ b/app/models/concerns/cacheable_attributes.rb
@@ -0,0 +1,74 @@
+module CacheableAttributes
+  extend ActiveSupport::Concern
+
+  included do
+    after_commit { self.class.expire }
+  end
+
+  class_methods do
+    def cache_key
+      "#{name}:#{Gitlab::VERSION}:#{Gitlab.migrations_hash}:#{Rails.version}".freeze
+    end
+
+    # Can be overriden
+    def current_without_cache
+      last
+    end
+
+    # Can be overriden
+    def defaults
+      {}
+    end
+
+    def build_from_defaults(attributes = {})
+      new(defaults.merge(attributes))
+    end
+
+    def cached
+      if RequestStore.active?
+        RequestStore[:"#{name}_cached_attributes"] ||= retrieve_from_cache
+      else
+        retrieve_from_cache
+      end
+    end
+
+    def retrieve_from_cache
+      record = Rails.cache.read(cache_key)
+      ensure_cache_setup if record.present?
+
+      record
+    end
+
+    def current
+      cached_record = cached
+      return cached_record if cached_record.present?
+
+      current_without_cache.tap { |current_record| current_record&.cache! }
+    rescue => e
+      if Rails.env.production?
+        Rails.logger.warn("Cached record for #{name} couldn't be loaded, falling back to uncached record: #{e}")
+      else
+        raise e
+      end
+      # Fall back to an uncached value if there are any problems (e.g. Redis down)
+      current_without_cache
+    end
+
+    def expire
+      Rails.cache.delete(cache_key)
+    rescue
+      # Gracefully handle when Redis is not available. For example,
+      # omnibus may fail here during gitlab:assets:compile.
+    end
+
+    def ensure_cache_setup
+      # This is a workaround for a Rails bug that causes attribute methods not
+      # to be loaded when read from cache: https://github.com/rails/rails/issues/27348
+      define_attribute_methods
+    end
+  end
+
+  def cache!
+    Rails.cache.write(self.class.cache_key, self)
+  end
+end
diff --git a/app/models/concerns/chronic_duration_attribute.rb b/app/models/concerns/chronic_duration_attribute.rb
new file mode 100644
index 00000000000..593a9b3d71d
--- /dev/null
+++ b/app/models/concerns/chronic_duration_attribute.rb
@@ -0,0 +1,39 @@
+module ChronicDurationAttribute
+  extend ActiveSupport::Concern
+
+  class_methods do
+    def chronic_duration_attr_reader(virtual_attribute, source_attribute)
+      define_method(virtual_attribute) do
+        chronic_duration_attributes[virtual_attribute] || output_chronic_duration_attribute(source_attribute)
+      end
+    end
+
+    def chronic_duration_attr_writer(virtual_attribute, source_attribute, parameters = {})
+      chronic_duration_attr_reader(virtual_attribute, source_attribute)
+
+      define_method("#{virtual_attribute}=") do |value|
+        chronic_duration_attributes[virtual_attribute] = value.presence || parameters[:default].presence.to_s
+
+        begin
+          new_value = value.present? ? ChronicDuration.parse(value).to_i : parameters[:default].presence
+          assign_attributes(source_attribute => new_value)
+        rescue ChronicDuration::DurationParseError
+          # ignore error as it will be caught by validation
+        end
+      end
+
+      validates virtual_attribute, allow_nil: true, duration: true
+    end
+
+    alias_method :chronic_duration_attr, :chronic_duration_attr_writer
+  end
+
+  def chronic_duration_attributes
+    @chronic_duration_attributes ||= {}
+  end
+
+  def output_chronic_duration_attribute(source_attribute)
+    value = attributes[source_attribute.to_s]
+    ChronicDuration.output(value, format: :short) if value
+  end
+end
diff --git a/app/models/concerns/deployment_platform.rb b/app/models/concerns/deployment_platform.rb
index faa94204e33..52851b3d0b2 100644
--- a/app/models/concerns/deployment_platform.rb
+++ b/app/models/concerns/deployment_platform.rb
@@ -1,16 +1,24 @@
 module DeploymentPlatform
-  # EE would override this and utilize the extra argument
+  # EE would override this and utilize environment argument
+  # rubocop:disable Gitlab/ModuleWithInstanceVariables
   def deployment_platform(environment: nil)
-    @deployment_platform ||=
-      find_cluster_platform_kubernetes ||
-      find_kubernetes_service_integration ||
-      build_cluster_and_deployment_platform
+    @deployment_platform ||= {}
+
+    @deployment_platform[environment] ||= find_deployment_platform(environment)
   end
 
   private
 
-  def find_cluster_platform_kubernetes
-    clusters.find_by(enabled: true)&.platform_kubernetes
+  def find_deployment_platform(environment)
+    find_cluster_platform_kubernetes(environment: environment) ||
+      find_kubernetes_service_integration ||
+      build_cluster_and_deployment_platform
+  end
+
+  # EE would override this and utilize environment argument
+  def find_cluster_platform_kubernetes(environment: nil)
+    clusters.enabled.default_environment
+      .last&.platform_kubernetes
   end
 
   def find_kubernetes_service_integration
diff --git a/app/models/concerns/diff_file.rb b/app/models/concerns/diff_file.rb
new file mode 100644
index 00000000000..72332072012
--- /dev/null
+++ b/app/models/concerns/diff_file.rb
@@ -0,0 +1,9 @@
+module DiffFile
+  extend ActiveSupport::Concern
+
+  def to_hash
+    keys = Gitlab::Git::Diff::SERIALIZE_KEYS - [:diff]
+
+    as_json(only: keys).merge(diff: diff).with_indifferent_access
+  end
+end
diff --git a/app/models/concerns/enum_with_nil.rb b/app/models/concerns/enum_with_nil.rb
new file mode 100644
index 00000000000..6b37903da20
--- /dev/null
+++ b/app/models/concerns/enum_with_nil.rb
@@ -0,0 +1,33 @@
+module EnumWithNil
+  extend ActiveSupport::Concern
+
+  included do
+    def self.enum_with_nil(definitions)
+      # use original `enum` to auto-define all methods
+      enum(definitions)
+
+      # override auto-defined methods only for the
+      # key which uses nil value
+      definitions.each do |name, values|
+        next unless key_with_nil = values.key(nil)
+
+        # E.g. for enum_with_nil failure_reason: { unknown_failure: nil }
+        # this overrides auto-generated method `unknown_failure?`
+        define_method("#{key_with_nil}?") do
+          Gitlab.rails5? ? self[name].nil? : super()
+        end
+
+        # E.g. for enum_with_nil failure_reason: { unknown_failure: nil }
+        # this overrides auto-generated method `failure_reason`
+        define_method(name) do
+          orig = super()
+
+          return orig unless Gitlab.rails5?
+          return orig unless orig.nil?
+
+          self.class.public_send(name.to_s.pluralize).key(nil) # rubocop:disable GitlabSecurity/PublicSend
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/concerns/fast_destroy_all.rb b/app/models/concerns/fast_destroy_all.rb
new file mode 100644
index 00000000000..7ea042c6742
--- /dev/null
+++ b/app/models/concerns/fast_destroy_all.rb
@@ -0,0 +1,91 @@
+##
+# This module is for replacing `dependent: :destroy` and `before_destroy` hooks.
+#
+# In general, `destroy_all` is inefficient because it calls each callback with `DELETE` queries i.e. O(n), whereas,
+# `delete_all` is efficient as it deletes all rows with a single `DELETE` query.
+#
+# It's better to use `delete_all` as our best practice, however,
+# if external data (e.g. ObjectStorage, FileStorage or Redis) are assosiated with database records,
+# it is difficult to accomplish it.
+#
+# This module defines a format to use `delete_all` and delete associated external data.
+# Here is an exmaple
+#
+# Situation
+# - `Project` has many `Ci::BuildTraceChunk` through `Ci::Build`
+# - `Ci::BuildTraceChunk` stores associated data in Redis, so it relies on `dependent: :destroy` and `before_destroy` for the deletion
+#
+# How to use
+# - Define `use_fast_destroy :build_trace_chunks` in `Project` model.
+# - Define `begin_fast_destroy` and `finalize_fast_destroy(params)` in `Ci::BuildTraceChunk` model.
+# - Use `fast_destroy_all` instead of `destroy` and `destroy_all`
+# - Remove `dependent: :destroy` and `before_destroy` as it's no longer need
+#
+# Expectation
+# - When a project is `destroy`ed, the associated trace_chunks will be deleted by `delete_all`,
+#   and the associated data will be removed, too.
+# - When `fast_destroy_all` is called, it also performns as same.
+module FastDestroyAll
+  extend ActiveSupport::Concern
+
+  ForbiddenActionError = Class.new(StandardError)
+
+  included do
+    before_destroy do
+      raise ForbiddenActionError, '`destroy` and `destroy_all` are forbbiden. Please use `fast_destroy_all`'
+    end
+  end
+
+  class_methods do
+    ##
+    # This method delete rows and associated external data efficiently
+    #
+    # This method can replace `destroy` and `destroy_all` without having `after_destroy` hook
+    def fast_destroy_all
+      params = begin_fast_destroy
+
+      delete_all
+
+      finalize_fast_destroy(params)
+    end
+
+    ##
+    # This method returns identifiers to delete associated external data (e.g. file paths, redis keys)
+    #
+    # This method must be defined in fast destroyable model
+    def begin_fast_destroy
+      raise NotImplementedError
+    end
+
+    ##
+    # This method deletes associated external data with the identifiers returned by `begin_fast_destroy`
+    #
+    # This method must be defined in fast destroyable model
+    def finalize_fast_destroy(params)
+      raise NotImplementedError
+    end
+  end
+
+  module Helpers
+    extend ActiveSupport::Concern
+
+    class_methods do
+      ##
+      # This method is to be defined on models which have fast destroyable models as children,
+      # and let us avoid to use `dependent: :destroy` hook
+      def use_fast_destroy(relation)
+        before_destroy(prepend: true) do
+          perform_fast_destroy(public_send(relation)) # rubocop:disable GitlabSecurity/PublicSend
+        end
+      end
+    end
+
+    def perform_fast_destroy(subject)
+      params = subject.begin_fast_destroy
+
+      run_after_commit do
+        subject.finalize_fast_destroy(params)
+      end
+    end
+  end
+end
diff --git a/app/models/concerns/group_descendant.rb b/app/models/concerns/group_descendant.rb
index 01957da0bf3..261ace57a17 100644
--- a/app/models/concerns/group_descendant.rb
+++ b/app/models/concerns/group_descendant.rb
@@ -37,7 +37,20 @@ module GroupDescendant
     parent ||= preloaded.detect { |possible_parent| possible_parent.is_a?(Group) && possible_parent.id == child.parent_id }
 
     if parent.nil? && !child.parent_id.nil?
-      raise ArgumentError.new('parent was not preloaded')
+      parent = child.parent
+
+      exception = ArgumentError.new <<~MSG
+        parent: [GroupDescendant: #{parent.inspect}] was not preloaded for [#{child.inspect}]")
+        This error is not user facing, but causes a +1 query.
+      MSG
+      extras = {
+        parent: parent,
+        child: child,
+        preloaded: preloaded.map(&:full_path)
+      }
+      issue_url = 'https://gitlab.com/gitlab-org/gitlab-ce/issues/40785'
+
+      Gitlab::Sentry.track_exception(exception, issue_url: issue_url, extra: extras)
     end
 
     if parent.nil? && hierarchy_top.present?
diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb
index 7c3ed96bc28..72c236a0fc7 100644
--- a/app/models/concerns/has_status.rb
+++ b/app/models/concerns/has_status.rb
@@ -11,6 +11,8 @@ module HasStatus
   STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3,
                     failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze
 
+  UnknownStatusError = Class.new(StandardError)
+
   class_methods do
     def status_sql
       scope_relevant = respond_to?(:exclude_ignored) ? exclude_ignored : all
diff --git a/app/models/concerns/has_variable.rb b/app/models/concerns/has_variable.rb
index 8a241e4374a..c8e20c0ab81 100644
--- a/app/models/concerns/has_variable.rb
+++ b/app/models/concerns/has_variable.rb
@@ -13,7 +13,7 @@ module HasVariable
     attr_encrypted :value,
        mode: :per_attribute_iv_and_salt,
        insecure_mode: true,
-       key: Gitlab::Application.secrets.db_key_base,
+       key: Settings.attr_encrypted_db_key_base,
        algorithm: 'aes-256-cbc'
 
     def key=(new_key)
diff --git a/app/models/concerns/iid_routes.rb b/app/models/concerns/iid_routes.rb
new file mode 100644
index 00000000000..246748cf52c
--- /dev/null
+++ b/app/models/concerns/iid_routes.rb
@@ -0,0 +1,9 @@
+module IidRoutes
+  ##
+  # This automagically enforces all related routes to use `iid` instead of `id`
+  # If you want to use `iid` for some routes and `id` for other routes, this module should not to be included,
+  # instead you should define `iid` or `id` explictly at each route generators. e.g. pipeline_path(project.id, pipeline.iid)
+  def to_param
+    iid.to_s
+  end
+end
diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb
index 5a566f3ac02..b93c1145f82 100644
--- a/app/models/concerns/issuable.rb
+++ b/app/models/concerns/issuable.rb
@@ -97,8 +97,6 @@ module Issuable
 
     strip_attributes :title
 
-    after_save :ensure_metrics, unless: :imported?
-
     # We want to use optimistic lock for cases when only title or description are involved
     # http://api.rubyonrails.org/classes/ActiveRecord/Locking/Optimistic.html
     def locking_enabled?
@@ -109,6 +107,10 @@ module Issuable
       false
     end
 
+    def etag_caching_enabled?
+      false
+    end
+
     def has_multiple_assignees?
       assignees.count > 1
     end
@@ -137,7 +139,7 @@ module Issuable
       fuzzy_search(query, [:title, :description])
     end
 
-    def sort(method, excluded_labels: [])
+    def sort_by_attribute(method, excluded_labels: [])
       sorted =
         case method.to_s
         when 'downvotes_desc'     then order_downvotes_desc
diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb
index caf8afa97f9..967fd9c5eea 100644
--- a/app/models/concerns/milestoneish.rb
+++ b/app/models/concerns/milestoneish.rb
@@ -45,11 +45,11 @@ module Milestoneish
   end
 
   def sorted_issues(user)
-    issues_visible_to_user(user).preload_associations.sort('label_priority')
+    issues_visible_to_user(user).preload_associations.sort_by_attribute('label_priority')
   end
 
   def sorted_merge_requests
-    merge_requests.sort('label_priority')
+    merge_requests.sort_by_attribute('label_priority')
   end
 
   def upcoming?
@@ -102,14 +102,14 @@ module Milestoneish
     Gitlab::TimeTrackingFormatter.output(total_issue_time_estimate)
   end
 
-  private
-
   def count_issues_by_state(user)
     memoize_per_user(user, :count_issues_by_state) do
       issues_visible_to_user(user).reorder(nil).group(:state).count
     end
   end
 
+  private
+
   def memoize_per_user(user, method_name)
     memoized_users[method_name][user&.id] ||= yield
   end
diff --git a/app/models/concerns/nonatomic_internal_id.rb b/app/models/concerns/nonatomic_internal_id.rb
deleted file mode 100644
index 9d0c9b8512f..00000000000
--- a/app/models/concerns/nonatomic_internal_id.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-module NonatomicInternalId
-  extend ActiveSupport::Concern
-
-  included do
-    validate :set_iid, on: :create
-    validates :iid, presence: true, numericality: true
-  end
-
-  def set_iid
-    if iid.blank?
-      parent = project || group
-      records = parent.public_send(self.class.name.tableize) # rubocop:disable GitlabSecurity/PublicSend
-      max_iid = records.maximum(:iid)
-
-      self.iid = max_iid.to_i + 1
-    end
-  end
-
-  def to_param
-    iid.to_s
-  end
-end
diff --git a/app/models/concerns/participable.rb b/app/models/concerns/participable.rb
index e48bc0be410..01b1ef9f82c 100644
--- a/app/models/concerns/participable.rb
+++ b/app/models/concerns/participable.rb
@@ -98,6 +98,10 @@ module Participable
 
     participants.merge(ext.users)
 
+    filter_by_ability(participants)
+  end
+
+  def filter_by_ability(participants)
     case self
     when PersonalSnippet
       Ability.users_that_can_read_personal_snippet(participants.to_a, self)
diff --git a/app/models/concerns/presentable.rb b/app/models/concerns/presentable.rb
index 7b33b837004..bc4fbd19a02 100644
--- a/app/models/concerns/presentable.rb
+++ b/app/models/concerns/presentable.rb
@@ -1,4 +1,12 @@
 module Presentable
+  extend ActiveSupport::Concern
+
+  class_methods do
+    def present(attributes)
+      all.map { |klass_object| klass_object.present(attributes) }
+    end
+  end
+
   def present(**attributes)
     Gitlab::View::Presenter::Factory
       .new(self, attributes)
diff --git a/app/models/concerns/project_features_compatibility.rb b/app/models/concerns/project_features_compatibility.rb
index b3fec99c816..1f7d78a2efe 100644
--- a/app/models/concerns/project_features_compatibility.rb
+++ b/app/models/concerns/project_features_compatibility.rb
@@ -1,4 +1,4 @@
-# Makes api V3 compatible with old project features permissions methods
+# Makes api V4 compatible with old project features permissions methods
 #
 # After migrating issues_enabled merge_requests_enabled builds_enabled snippets_enabled and wiki_enabled
 # fields to a new table "project_features", support for the old fields is still needed in the API.
diff --git a/app/models/concerns/protected_ref.rb b/app/models/concerns/protected_ref.rb
index 454374121f3..94eef4ff7cd 100644
--- a/app/models/concerns/protected_ref.rb
+++ b/app/models/concerns/protected_ref.rb
@@ -31,7 +31,7 @@ module ProtectedRef
       end
     end
 
-    def protected_ref_accessible_to?(ref, user, action:, protected_refs: nil)
+    def protected_ref_accessible_to?(ref, user, project:, action:, protected_refs: nil)
       access_levels_for_ref(ref, action: action, protected_refs: protected_refs).any? do |access_level|
         access_level.check_access(user)
       end
diff --git a/app/models/concerns/protected_ref_access.rb b/app/models/concerns/protected_ref_access.rb
index bfda5b1678b..e3a7f2d5498 100644
--- a/app/models/concerns/protected_ref_access.rb
+++ b/app/models/concerns/protected_ref_access.rb
@@ -8,8 +8,8 @@ module ProtectedRefAccess
   ].freeze
 
   HUMAN_ACCESS_LEVELS = {
-    Gitlab::Access::MASTER => "Masters".freeze,
-    Gitlab::Access::DEVELOPER => "Developers + Masters".freeze,
+    Gitlab::Access::MASTER => "Maintainers".freeze,
+    Gitlab::Access::DEVELOPER => "Developers + Maintainers".freeze,
     Gitlab::Access::NO_ACCESS => "No one".freeze
   }.freeze
 
diff --git a/app/models/concerns/reactive_caching.rb b/app/models/concerns/reactive_caching.rb
index 2589215ad19..be0a5b49012 100644
--- a/app/models/concerns/reactive_caching.rb
+++ b/app/models/concerns/reactive_caching.rb
@@ -60,22 +60,26 @@ module ReactiveCaching
     end
 
     def with_reactive_cache(*args, &blk)
-      within_reactive_cache_lifetime(*args) do
+      bootstrap = !within_reactive_cache_lifetime?(*args)
+      Rails.cache.write(alive_reactive_cache_key(*args), true, expires_in: self.class.reactive_cache_lifetime)
+
+      if bootstrap
+        ReactiveCachingWorker.perform_async(self.class, id, *args)
+        nil
+      else
         data = Rails.cache.read(full_reactive_cache_key(*args))
         yield data if data.present?
       end
-    ensure
-      Rails.cache.write(alive_reactive_cache_key(*args), true, expires_in: self.class.reactive_cache_lifetime)
-      ReactiveCachingWorker.perform_async(self.class, id, *args)
     end
 
     def clear_reactive_cache!(*args)
       Rails.cache.delete(full_reactive_cache_key(*args))
+      Rails.cache.delete(alive_reactive_cache_key(*args))
     end
 
     def exclusively_update_reactive_cache!(*args)
       locking_reactive_cache(*args) do
-        within_reactive_cache_lifetime(*args) do
+        if within_reactive_cache_lifetime?(*args)
           enqueuing_update(*args) do
             value = calculate_reactive_cache(*args)
             Rails.cache.write(full_reactive_cache_key(*args), value)
@@ -105,8 +109,8 @@ module ReactiveCaching
       Gitlab::ExclusiveLease.cancel(full_reactive_cache_key(*args), uuid)
     end
 
-    def within_reactive_cache_lifetime(*args)
-      yield if Rails.cache.read(alive_reactive_cache_key(*args))
+    def within_reactive_cache_lifetime?(*args)
+      !!Rails.cache.read(alive_reactive_cache_key(*args))
     end
 
     def enqueuing_update(*args)
diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb
index b889f4202dc..3bdc1330d23 100644
--- a/app/models/concerns/redis_cacheable.rb
+++ b/app/models/concerns/redis_cacheable.rb
@@ -7,7 +7,11 @@ module RedisCacheable
   class_methods do
     def cached_attr_reader(*attributes)
       attributes.each do |attribute|
-        define_method("#{attribute}") do
+        define_method(attribute) do
+          unless self.has_attribute?(attribute)
+            raise ArgumentError, "`cached_attr_reader` requires the #{self.class.name}\##{attribute} attribute to have a database column"
+          end
+
           cached_attribute(attribute) || read_attribute(attribute)
         end
       end
@@ -15,13 +19,16 @@ module RedisCacheable
   end
 
   def cached_attribute(attribute)
-    (cached_attributes || {})[attribute]
+    cached_value = (cached_attributes || {})[attribute]
+    cast_value_from_cache(attribute, cached_value) if cached_value
   end
 
   def cache_attributes(values)
     Gitlab::Redis::SharedState.with do |redis|
       redis.set(cache_attribute_key, values.to_json, ex: CACHED_ATTRIBUTES_EXPIRY_TIME)
     end
+
+    clear_memoization(:cached_attributes)
   end
 
   private
@@ -38,4 +45,12 @@ module RedisCacheable
       end
     end
   end
+
+  def cast_value_from_cache(attribute, value)
+    if Gitlab.rails5?
+      self.class.type_for_attribute(attribute.to_s).cast(value)
+    else
+      self.class.column_for_attribute(attribute).type_cast_from_database(value)
+    end
+  end
 end
diff --git a/app/models/concerns/resolvable_note.rb b/app/models/concerns/resolvable_note.rb
index 668c5a079e3..4a0f8b92b3a 100644
--- a/app/models/concerns/resolvable_note.rb
+++ b/app/models/concerns/resolvable_note.rb
@@ -32,7 +32,7 @@ module ResolvableNote
 
   # Keep this method in sync with the `potentially_resolvable` scope
   def potentially_resolvable?
-    RESOLVABLE_TYPES.include?(self.class.name) && noteable.supports_resolvable_notes?
+    RESOLVABLE_TYPES.include?(self.class.name) && noteable&.supports_resolvable_notes?
   end
 
   # Keep this method in sync with the `resolvable` scope
diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb
index dfd7d94450b..0176a12a131 100644
--- a/app/models/concerns/routable.rb
+++ b/app/models/concerns/routable.rb
@@ -4,7 +4,9 @@ module Routable
   extend ActiveSupport::Concern
 
   included do
-    has_one :route, as: :source, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+    # Remove `inverse_of: source` when upgraded to rails 5.2
+    # See https://github.com/rails/rails/pull/28808
+    has_one :route, as: :source, autosave: true, dependent: :destroy, inverse_of: :source # rubocop:disable Cop/ActiveRecordDependent
     has_many :redirect_routes, as: :source, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
     validates :route, presence: true
@@ -102,7 +104,7 @@ module Routable
   # the route. Caching this per request ensures that even if we have multiple instances,
   # we will not have to duplicate work, avoiding N+1 queries in some cases.
   def full_path
-    return uncached_full_path unless RequestStore.active?
+    return uncached_full_path unless RequestStore.active? && persisted?
 
     RequestStore[full_path_key] ||= uncached_full_path
   end
@@ -124,6 +126,11 @@ module Routable
     end
   end
 
+  # Group would override this to check from association
+  def owned_by?(user)
+    owner == user
+  end
+
   private
 
   def set_path_errors
diff --git a/app/models/concerns/sha_attribute.rb b/app/models/concerns/sha_attribute.rb
index 703a72c355c..3796737427a 100644
--- a/app/models/concerns/sha_attribute.rb
+++ b/app/models/concerns/sha_attribute.rb
@@ -4,18 +4,34 @@ module ShaAttribute
   module ClassMethods
     def sha_attribute(name)
       return if ENV['STATIC_VERIFICATION']
-      return unless table_exists?
+
+      validate_binary_column_exists!(name) unless Rails.env.production?
+
+      attribute(name, Gitlab::Database::ShaAttribute.new)
+    end
+
+    # This only gets executed in non-production environments as an additional check to ensure
+    # the column is the correct type.  In production it should behave like any other attribute.
+    # See https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/5502 for more discussion
+    def validate_binary_column_exists!(name)
+      unless table_exists?
+        warn "WARNING: sha_attribute #{name.inspect} is invalid since the table doesn't exist - you may need to run database migrations"
+        return
+      end
 
       column = columns.find { |c| c.name == name.to_s }
 
-      # In case the table doesn't exist we won't be able to find the column,
-      # thus we will only check the type if the column is present.
-      if column && column.type != :binary
-        raise ArgumentError,
-          "sha_attribute #{name.inspect} is invalid since the column type is not :binary"
+      unless column
+        warn "WARNING: sha_attribute #{name.inspect} is invalid since the column doesn't exist - you may need to run database migrations"
+        return
       end
 
-      attribute(name, Gitlab::Database::ShaAttribute.new)
+      unless column.type == :binary
+        raise ArgumentError.new("sha_attribute #{name.inspect} is invalid since the column type is not :binary")
+      end
+    rescue => error
+      Gitlab::AppLogger.error "ShaAttribute initialization: #{error.message}"
+      raise
     end
   end
 end
diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb
index cefa5c13c5f..cb76ae971d4 100644
--- a/app/models/concerns/sortable.rb
+++ b/app/models/concerns/sortable.rb
@@ -12,8 +12,8 @@ module Sortable
     scope :order_created_asc, -> { reorder(created_at: :asc) }
     scope :order_updated_desc, -> { reorder(updated_at: :desc) }
     scope :order_updated_asc, -> { reorder(updated_at: :asc) }
-    scope :order_name_asc, -> { reorder(name: :asc) }
-    scope :order_name_desc, -> { reorder(name: :desc) }
+    scope :order_name_asc, -> { reorder(Arel::Nodes::Ascending.new(arel_table[:name].lower)) }
+    scope :order_name_desc, -> { reorder(Arel::Nodes::Descending.new(arel_table[:name].lower)) }
   end
 
   module ClassMethods
diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb
index f05e606995d..f66bdd529f1 100644
--- a/app/models/concerns/storage/legacy_namespace.rb
+++ b/app/models/concerns/storage/legacy_namespace.rb
@@ -45,25 +45,25 @@ module Storage
 
     # Hooks
 
-    # Save the storage paths before the projects are destroyed to use them on after destroy
+    # Save the storages before the projects are destroyed to use them on after destroy
     def prepare_for_destroy
-      old_repository_storage_paths
+      old_repository_storages
     end
 
     private
 
     def move_repositories
-      # Move the namespace directory in all storage paths used by member projects
-      repository_storage_paths.each do |repository_storage_path|
+      # Move the namespace directory in all storages used by member projects
+      repository_storages.each do |repository_storage|
         # Ensure old directory exists before moving it
-        gitlab_shell.add_namespace(repository_storage_path, full_path_was)
+        gitlab_shell.add_namespace(repository_storage, full_path_was)
 
         # Ensure new directory exists before moving it (if there's a parent)
-        gitlab_shell.add_namespace(repository_storage_path, parent.full_path) if parent
+        gitlab_shell.add_namespace(repository_storage, parent.full_path) if parent
 
-        unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path)
+        unless gitlab_shell.mv_namespace(repository_storage, full_path_was, full_path)
 
-          Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}"
+          Rails.logger.error "Exception moving path #{repository_storage} from #{full_path_was} to #{full_path}"
 
           # if we cannot move namespace directory we should rollback
           # db changes in order to prevent out of sync between db and fs
@@ -72,33 +72,33 @@ module Storage
       end
     end
 
-    def old_repository_storage_paths
-      @old_repository_storage_paths ||= repository_storage_paths
+    def old_repository_storages
+      @old_repository_storage_paths ||= repository_storages
     end
 
-    def repository_storage_paths
+    def repository_storages
       # We need to get the storage paths for all the projects, even the ones that are
       # pending delete. Unscoping also get rids of the default order, which causes
       # problems with SELECT DISTINCT.
       Project.unscoped do
-        all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
+        all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage)
       end
     end
 
     def rm_dir
       # Remove the namespace directory in all storages paths used by member projects
-      old_repository_storage_paths.each do |repository_storage_path|
+      old_repository_storages.each do |repository_storage|
         # Move namespace directory into trash.
         # We will remove it later async
         new_path = "#{full_path}+#{id}+deleted"
 
-        if gitlab_shell.mv_namespace(repository_storage_path, full_path, new_path)
+        if gitlab_shell.mv_namespace(repository_storage, full_path, new_path)
           Gitlab::AppLogger.info %Q(Namespace directory "#{full_path}" moved to "#{new_path}")
 
           # Remove namespace directroy async with delay so
           # GitLab has time to remove all projects first
           run_after_commit do
-            GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
+            GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage, new_path)
           end
         end
       end
diff --git a/app/models/concerns/time_trackable.rb b/app/models/concerns/time_trackable.rb
index 5911b56c34c..0fc321c52bc 100644
--- a/app/models/concerns/time_trackable.rb
+++ b/app/models/concerns/time_trackable.rb
@@ -51,6 +51,10 @@ module TimeTrackable
     Gitlab::TimeTrackingFormatter.output(time_estimate)
   end
 
+  def time_estimate=(val)
+    val.is_a?(Integer) ? super([val, Gitlab::Database::MAX_INT_VALUE].min) : super(val)
+  end
+
   private
 
   def reset_spent_time
diff --git a/app/models/concerns/uniquify.rb b/app/models/concerns/uniquify.rb
index a7fe5951b6e..549a76da20e 100644
--- a/app/models/concerns/uniquify.rb
+++ b/app/models/concerns/uniquify.rb
@@ -1,13 +1,21 @@
+# Uniquify
+#
+# Return a version of the given 'base' string that is unique
+# by appending a counter to it. Uniqueness is determined by
+# repeated calls to the passed block.
+#
+# You can pass an initial value for the counter, if not given
+# counting starts from 1.
+#
+# If `base` is a function/proc, we expect that calling it with a
+# candidate counter returns a string to test/return.
 class Uniquify
-  # Return a version of the given 'base' string that is unique
-  # by appending a counter to it. Uniqueness is determined by
-  # repeated calls to the passed block.
-  #
-  # If `base` is a function/proc, we expect that calling it with a
-  # candidate counter returns a string to test/return.
+  def initialize(counter = nil)
+    @counter = counter
+  end
+
   def string(base)
     @base = base
-    @counter = nil
 
     increment_counter! while yield(base_string)
     base_string
diff --git a/app/models/concerns/with_uploads.rb b/app/models/concerns/with_uploads.rb
new file mode 100644
index 00000000000..4245d083a49
--- /dev/null
+++ b/app/models/concerns/with_uploads.rb
@@ -0,0 +1,43 @@
+# Mounted uploaders are destroyed by carrierwave's after_commit
+# hook. This hook fetches upload location (local vs remote) from
+# Upload model. So it's neccessary to make sure that during that
+# after_commit hook model's associated uploads are not deleted yet.
+# IOW we can not use dependent: :destroy :
+# has_many :uploads, as: :model, dependent: :destroy
+#
+# And because not-mounted uploads require presence of upload's
+# object model when destroying them (FileUploader's `build_upload` method
+# references `model` on delete), we can not use after_commit hook for these
+# uploads.
+#
+# Instead FileUploads are destroyed in before_destroy hook and remaining uploads
+# are destroyed by the carrierwave's after_commit hook.
+
+module WithUploads
+  extend ActiveSupport::Concern
+
+  # Currently there is no simple way how to select only not-mounted
+  # uploads, it should be all FileUploaders so we select them by
+  # `uploader` class
+  FILE_UPLOADERS = %w(PersonalFileUploader NamespaceFileUploader FileUploader).freeze
+
+  included do
+    has_many :uploads, as: :model
+
+    before_destroy :destroy_file_uploads
+  end
+
+  # mounted uploads are deleted in carrierwave's after_commit hook,
+  # but FileUploaders which are not mounted must be deleted explicitly and
+  # it can not be done in after_commit because FileUploader requires loads
+  # associated model on destroy (which is already deleted in after_commit)
+  def destroy_file_uploads
+    self.uploads.where(uploader: FILE_UPLOADERS).find_each do |upload|
+      upload.destroy
+    end
+  end
+
+  def retrieve_upload(_identifier, paths)
+    uploads.find_by(path: paths)
+  end
+end