Merge remote-tracking branch 'dev/master' into security-encrypt-runners-tokens

* dev/master: (302 commits)

Conflicts:
	db/schema.rb
	lib/gitlab/import_export/import_export.yml

32 files changed, 395 insertions, 178 deletions

diff --git a/app/models/application_record.rb b/app/models/application_record.rb
new file mode 100644
index 00000000000..71fbba5b328
--- /dev/null
+++ b/app/models/application_record.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class ApplicationRecord < ActiveRecord::Base
+  self.abstract_class = true
+end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 889f8ce27a6..d60861dc95f 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -98,7 +98,7 @@ module Ci
 
     scope :matches_tag_ids, -> (tag_ids) do
       matcher = ::ActsAsTaggableOn::Tagging
-        .where(taggable_type: CommitStatus)
+        .where(taggable_type: CommitStatus.name)
         .where(context: 'tags')
         .where('taggable_id = ci_builds.id')
         .where.not(tag_id: tag_ids).select('1')
@@ -108,7 +108,7 @@ module Ci
 
     scope :with_any_tags, -> do
       matcher = ::ActsAsTaggableOn::Tagging
-        .where(taggable_type: CommitStatus)
+        .where(taggable_type: CommitStatus.name)
         .where(context: 'tags')
         .where('taggable_id = ci_builds.id').select('1')
 
@@ -464,7 +464,9 @@ module Ci
     end
 
     def repo_url
-      auth = "gitlab-ci-token:#{ensure_token!}@"
+      return unless token
+
+      auth = "gitlab-ci-token:#{token}@"
       project.http_url_to_repo.sub(%r{^https?://}) do |prefix|
         prefix + auth
       end
@@ -725,7 +727,7 @@ module Ci
 
       trace = trace.dup
       Gitlab::Ci::MaskSecret.mask!(trace, project.runners_token) if project
-      Gitlab::Ci::MaskSecret.mask!(trace, token)
+      Gitlab::Ci::MaskSecret.mask!(trace, token) if token
       trace
     end
 
@@ -814,12 +816,12 @@ module Ci
           .concat(pipeline.persisted_variables)
           .append(key: 'CI_JOB_ID', value: id.to_s)
           .append(key: 'CI_JOB_URL', value: Gitlab::Routing.url_helpers.project_job_url(project, self))
-          .append(key: 'CI_JOB_TOKEN', value: token, public: false)
+          .append(key: 'CI_JOB_TOKEN', value: token.to_s, public: false)
           .append(key: 'CI_BUILD_ID', value: id.to_s)
-          .append(key: 'CI_BUILD_TOKEN', value: token, public: false)
+          .append(key: 'CI_BUILD_TOKEN', value: token.to_s, public: false)
           .append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER)
-          .append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false)
-          .append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false)
+          .append(key: 'CI_REGISTRY_PASSWORD', value: token.to_s, public: false)
+          .append(key: 'CI_REPOSITORY_URL', value: repo_url.to_s, public: false)
           .concat(deploy_token_variables)
       end
     end
@@ -831,9 +833,9 @@ module Ci
         variables.append(key: 'GITLAB_FEATURES', value: project.licensed_features.join(','))
         variables.append(key: 'CI_SERVER_NAME', value: 'GitLab')
         variables.append(key: 'CI_SERVER_VERSION', value: Gitlab::VERSION)
-        variables.append(key: 'CI_SERVER_VERSION_MAJOR', value: gitlab_version_info.major.to_s)
-        variables.append(key: 'CI_SERVER_VERSION_MINOR', value: gitlab_version_info.minor.to_s)
-        variables.append(key: 'CI_SERVER_VERSION_PATCH', value: gitlab_version_info.patch.to_s)
+        variables.append(key: 'CI_SERVER_VERSION_MAJOR', value: Gitlab.version_info.major.to_s)
+        variables.append(key: 'CI_SERVER_VERSION_MINOR', value: Gitlab.version_info.minor.to_s)
+        variables.append(key: 'CI_SERVER_VERSION_PATCH', value: Gitlab.version_info.patch.to_s)
         variables.append(key: 'CI_SERVER_REVISION', value: Gitlab.revision)
         variables.append(key: 'CI_JOB_NAME', value: name)
         variables.append(key: 'CI_JOB_STAGE', value: stage)
@@ -850,10 +852,6 @@ module Ci
       end
     end
 
-    def gitlab_version_info
-      @gitlab_version_info ||= Gitlab::VersionInfo.parse(Gitlab::VERSION)
-    end
-
     def legacy_variables
       Gitlab::Ci::Variables::Collection.new.tap do |variables|
         variables.append(key: 'CI_BUILD_REF', value: sha)
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 56010e899a4..9512ba42f67 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -58,15 +58,9 @@ module Ci
 
     after_create :keep_around_commits, unless: :importing?
 
-    enum_with_nil source: {
-      unknown: nil,
-      push: 1,
-      web: 2,
-      trigger: 3,
-      schedule: 4,
-      api: 5,
-      external: 6
-    }
+    # We use `Ci::PipelineEnums.sources` here so that EE can more easily extend
+    # this `Hash` with new values.
+    enum_with_nil source: ::Ci::PipelineEnums.sources
 
     enum_with_nil config_source: {
       unknown_source: nil,
@@ -74,10 +68,9 @@ module Ci
       auto_devops_source: 2
     }
 
-    enum failure_reason: {
-      unknown_failure: 0,
-      config_error: 1
-    }
+    # We use `Ci::PipelineEnums.failure_reasons` here so that EE can more easily
+    # extend this `Hash` with new values.
+    enum failure_reason: ::Ci::PipelineEnums.failure_reasons
 
     state_machine :status, initial: :created do
       event :enqueue do
diff --git a/app/models/ci/pipeline_enums.rb b/app/models/ci/pipeline_enums.rb
new file mode 100644
index 00000000000..8d8d16e2ec1
--- /dev/null
+++ b/app/models/ci/pipeline_enums.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Ci
+  module PipelineEnums
+    # Returns the `Hash` to use for creating the `failure_reason` enum for
+    # `Ci::Pipeline`.
+    def self.failure_reasons
+      {
+        unknown_failure: 0,
+        config_error: 1
+      }
+    end
+
+    # Returns the `Hash` to use for creating the `sources` enum for
+    # `Ci::Pipeline`.
+    def self.sources
+      {
+        unknown: nil,
+        push: 1,
+        web: 2,
+        trigger: 3,
+        schedule: 4,
+        api: 5,
+        external: 6
+      }
+    end
+  end
+end
diff --git a/app/models/clusters/applications/knative.rb b/app/models/clusters/applications/knative.rb
index a79a97576d1..c66d5ce54db 100644
--- a/app/models/clusters/applications/knative.rb
+++ b/app/models/clusters/applications/knative.rb
@@ -41,6 +41,10 @@ module Clusters
         )
       end
 
+      def client
+        cluster.platform_kubernetes.kubeclient.knative_client
+      end
+
       private
 
       def install_script
diff --git a/app/models/clusters/concerns/application_status.rb b/app/models/clusters/concerns/application_status.rb
index 93bdf9c223d..0e74cce29b7 100644
--- a/app/models/clusters/concerns/application_status.rb
+++ b/app/models/clusters/concerns/application_status.rb
@@ -64,6 +64,13 @@ module Clusters
             status_reason = transition.args.first
             app_status.status_reason = status_reason if status_reason
           end
+
+          before_transition any => [:installed, :updated] do |app_status, _|
+            # When installing any application we are also performing an update
+            # of tiller (see Gitlab::Kubernetes::Helm::ClientCommand) so
+            # therefore we need to reflect that in the database.
+            app_status.cluster.application_helm.update!(version: Gitlab::Kubernetes::Helm::HELM_VERSION)
+          end
         end
       end
 
diff --git a/app/models/clusters/kubernetes_namespace.rb b/app/models/clusters/kubernetes_namespace.rb
index ac7f9193b87..34f5e38ff79 100644
--- a/app/models/clusters/kubernetes_namespace.rb
+++ b/app/models/clusters/kubernetes_namespace.rb
@@ -11,9 +11,13 @@ module Clusters
     belongs_to :project, class_name: '::Project'
     has_one :platform_kubernetes, through: :cluster
 
+    before_validation :set_defaults
+
     validates :namespace, presence: true
     validates :namespace, uniqueness: { scope: :cluster_id }
 
+    validates :service_account_name, presence: true
+
     delegate :ca_pem, to: :platform_kubernetes, allow_nil: true
     delegate :api_url, to: :platform_kubernetes, allow_nil: true
 
@@ -22,42 +26,49 @@ module Clusters
         key: Settings.attr_encrypted_db_key_base_truncated,
         algorithm: 'aes-256-cbc'
 
+    scope :has_service_account_token, -> { where.not(encrypted_service_account_token: nil) }
+
     def token_name
       "#{namespace}-token"
     end
 
-    def configure_predefined_credentials
-      self.namespace = kubernetes_or_project_namespace
-      self.service_account_name = default_service_account_name
-    end
-
     def predefined_variables
       config = YAML.dump(kubeconfig)
 
       Gitlab::Ci::Variables::Collection.new.tap do |variables|
         variables
-          .append(key: 'KUBE_SERVICE_ACCOUNT', value: service_account_name)
-          .append(key: 'KUBE_NAMESPACE', value: namespace)
-          .append(key: 'KUBE_TOKEN', value: service_account_token, public: false)
+          .append(key: 'KUBE_SERVICE_ACCOUNT', value: service_account_name.to_s)
+          .append(key: 'KUBE_NAMESPACE', value: namespace.to_s)
+          .append(key: 'KUBE_TOKEN', value: service_account_token.to_s, public: false)
           .append(key: 'KUBECONFIG', value: config, public: false, file: true)
       end
     end
 
-    private
-
-    def kubernetes_or_project_namespace
-      platform_kubernetes&.namespace.presence || project_namespace
+    def set_defaults
+      self.namespace ||= default_platform_kubernetes_namespace
+      self.namespace ||= default_project_namespace
+      self.service_account_name ||= default_service_account_name
     end
 
+    private
+
     def default_service_account_name
+      return unless namespace
+
       "#{namespace}-service-account"
     end
 
-    def project_namespace
-      Gitlab::NamespaceSanitizer.sanitize(project_slug)
+    def default_platform_kubernetes_namespace
+      platform_kubernetes&.namespace.presence
+    end
+
+    def default_project_namespace
+      Gitlab::NamespaceSanitizer.sanitize(project_slug) if project_slug
     end
 
     def project_slug
+      return unless project
+
       "#{project.path}-#{project.id}".downcase
     end
 
diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb
index ea02ae6c9d8..3c5d7756eec 100644
--- a/app/models/clusters/platforms/kubernetes.rb
+++ b/app/models/clusters/platforms/kubernetes.rb
@@ -83,7 +83,7 @@ module Clusters
               .append(key: 'KUBE_CA_PEM_FILE', value: ca_pem, file: true)
           end
 
-          if kubernetes_namespace = cluster.kubernetes_namespaces.find_by(project: project)
+          if kubernetes_namespace = cluster.kubernetes_namespaces.has_service_account_token.find_by(project: project)
             variables.concat(kubernetes_namespace.predefined_variables)
           else
             # From 11.5, every Clusters::Project should have at least one
@@ -173,9 +173,7 @@ module Clusters
         kubeclient = build_kube_client!
 
         kubeclient.get_pods(namespace: actual_namespace).as_json
-      rescue Kubeclient::HttpError => err
-        raise err unless err.error_code == 404
-
+      rescue Kubeclient::ResourceNotFoundError
         []
       end
 
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 9dd0cbacd9e..546fcc54a15 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -230,24 +230,13 @@ class Commit
 
   def lazy_author
     BatchLoader.for(author_email.downcase).batch do |emails, loader|
-      # A Hash that maps user Emails to the corresponding User objects. The
-      # Emails at this point are the _primary_ Emails of the Users.
-      users_for_emails = User
-        .by_any_email(emails)
-        .each_with_object({}) { |user, hash| hash[user.email] = user }
-
-      users_for_ids = users_for_emails
-        .values
-        .each_with_object({}) { |user, hash| hash[user.id] = user }
-
-      # Some commits may have used an alternative Email address. In this case we
-      # need to query the "emails" table to map those addresses to User objects.
-      Email
-        .where(email: emails - users_for_emails.keys)
-        .pluck(:email, :user_id)
-        .each { |(email, id)| users_for_emails[email] = users_for_ids[id] }
-
-      users_for_emails.each { |email, user| loader.call(email, user) }
+      users = User.by_any_email(emails).includes(:emails)
+
+      emails.each do |email|
+        user = users.find { |u| u.any_email?(email) }
+
+        loader.call(email, user)
+      end
     end
   end
 
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 755f8bd4d06..0f50bd39131 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -42,18 +42,9 @@ class CommitStatus < ActiveRecord::Base
   scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) }
   scope :after_stage, -> (index) { where('stage_idx > ?', index) }
 
-  enum_with_nil failure_reason: {
-    unknown_failure: nil,
-    script_failure: 1,
-    api_failure: 2,
-    stuck_or_timeout_failure: 3,
-    runner_system_failure: 4,
-    missing_dependency_failure: 5,
-    runner_unsupported: 6,
-    stale_schedule: 7,
-    job_execution_timeout: 8,
-    archived_failure: 9
-  }
+  # We use `CommitStatusEnums.failure_reasons` here so that EE can more easily
+  # extend this `Hash` with new values.
+  enum_with_nil failure_reason: ::CommitStatusEnums.failure_reasons
 
   ##
   # We still create some CommitStatuses outside of CreatePipelineService.
diff --git a/app/models/commit_status_enums.rb b/app/models/commit_status_enums.rb
new file mode 100644
index 00000000000..152105d9429
--- /dev/null
+++ b/app/models/commit_status_enums.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module CommitStatusEnums
+  # Returns the Hash to use for creating the `failure_reason` enum for
+  # `CommitStatus`.
+  def self.failure_reasons
+    {
+      unknown_failure: nil,
+      script_failure: 1,
+      api_failure: 2,
+      stuck_or_timeout_failure: 3,
+      runner_system_failure: 4,
+      missing_dependency_failure: 5,
+      runner_unsupported: 6,
+      stale_schedule: 7,
+      job_execution_timeout: 8,
+      archived_failure: 9
+    }
+  end
+end
diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb
index 0d5311a9985..b42236c1fa2 100644
--- a/app/models/concerns/avatarable.rb
+++ b/app/models/concerns/avatarable.rb
@@ -86,7 +86,7 @@ module Avatarable
         params[:model].upload_paths(params[:identifier])
       end
 
-      Upload.where(uploader: AvatarUploader, path: paths).find_each do |upload|
+      Upload.where(uploader: AvatarUploader.name, path: paths).find_each do |upload|
         model = model_class.instantiate('id' => upload.model_id)
 
         loader.call({ model: model, identifier: File.basename(upload.path) }, upload)
diff --git a/app/models/concerns/deployable.rb b/app/models/concerns/deployable.rb
index 85db01af18d..bc12b06b5af 100644
--- a/app/models/concerns/deployable.rb
+++ b/app/models/concerns/deployable.rb
@@ -13,6 +13,10 @@ module Deployable
         name: expanded_environment_name
       )
 
+      # If we failed to persist envirionment record by validation error, such as name with invalid character,
+      # the job will fall back to a non-environment job.
+      return unless environment.persisted?
+
       create_deployment!(
         project_id: environment.project_id,
         environment: environment,
diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb
index 298d0d42d90..0d88b34fb48 100644
--- a/app/models/concerns/mentionable.rb
+++ b/app/models/concerns/mentionable.rb
@@ -97,9 +97,9 @@ module Mentionable
   # Allows heavy processing to be skipped
   def matches_cross_reference_regex?
     reference_pattern = if !project || project.default_issues_tracker?
-                          ReferenceRegexes::DEFAULT_PATTERN
+                          ReferenceRegexes.default_pattern
                         else
-                          ReferenceRegexes::EXTERNAL_PATTERN
+                          ReferenceRegexes.external_pattern
                         end
 
     self.class.mentionable_attrs.any? do |attr, _|
diff --git a/app/models/concerns/mentionable/reference_regexes.rb b/app/models/concerns/mentionable/reference_regexes.rb
index fe8fbb71184..b8fb3f71925 100644
--- a/app/models/concerns/mentionable/reference_regexes.rb
+++ b/app/models/concerns/mentionable/reference_regexes.rb
@@ -2,6 +2,8 @@
 
 module Mentionable
   module ReferenceRegexes
+    extend Gitlab::Utils::StrongMemoize
+
     def self.reference_pattern(link_patterns, issue_pattern)
       Regexp.union(link_patterns,
                    issue_pattern,
@@ -15,16 +17,20 @@ module Mentionable
       ]
     end
 
-    DEFAULT_PATTERN = begin
-      issue_pattern = Issue.reference_pattern
-      link_patterns = Regexp.union([Issue, Commit, MergeRequest, Epic].map(&:link_reference_pattern).compact)
-      reference_pattern(link_patterns, issue_pattern)
+    def self.default_pattern
+      strong_memoize(:default_pattern) do
+        issue_pattern = Issue.reference_pattern
+        link_patterns = Regexp.union([Issue, Commit, MergeRequest, Epic].map(&:link_reference_pattern).compact)
+        reference_pattern(link_patterns, issue_pattern)
+      end
     end
 
-    EXTERNAL_PATTERN = begin
-      issue_pattern = IssueTrackerService.reference_pattern
-      link_patterns = URI.regexp(%w(http https))
-      reference_pattern(link_patterns, issue_pattern)
+    def self.external_pattern
+      strong_memoize(:external_pattern) do
+        issue_pattern = IssueTrackerService.reference_pattern
+        link_patterns = URI.regexp(%w(http https))
+        reference_pattern(link_patterns, issue_pattern)
+      end
     end
   end
 end
diff --git a/app/models/concerns/mirror_authentication.rb b/app/models/concerns/mirror_authentication.rb
new file mode 100644
index 00000000000..e3e1a0441f8
--- /dev/null
+++ b/app/models/concerns/mirror_authentication.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+# Mirroring may use password or SSH public-key authentication. This concern
+# implements support for persisting the necessary data in a `credentials`
+# serialized attribute. It also needs an `url` method to be defined
+module MirrorAuthentication
+  SSH_PRIVATE_KEY_OPTS = {
+    type: 'RSA',
+    bits: 4096
+  }.freeze
+
+  extend ActiveSupport::Concern
+
+  included do
+    validates :auth_method, inclusion: { in: %w[password ssh_public_key] }, allow_blank: true
+
+    # We should generate a key even if there's no SSH URL present
+    before_validation :generate_ssh_private_key!, if: -> {
+      regenerate_ssh_private_key || ( auth_method == 'ssh_public_key' && ssh_private_key.blank? )
+    }
+
+    credentials_field :auth_method, reader: false
+    credentials_field :ssh_known_hosts
+    credentials_field :ssh_known_hosts_verified_at
+    credentials_field :ssh_known_hosts_verified_by_id
+    credentials_field :ssh_private_key
+    credentials_field :user
+    credentials_field :password
+  end
+
+  class_methods do
+    def credentials_field(name, reader: true)
+      if reader
+        define_method(name) do
+          credentials[name] if credentials.present?
+        end
+      end
+
+      define_method("#{name}=") do |value|
+        self.credentials ||= {}
+
+        # Removal of the password, username, etc, generally causes an update of
+        # the value to the empty string. Detect and gracefully handle this case.
+        if value.present?
+          self.credentials[name] = value
+        else
+          self.credentials.delete(name)
+        end
+      end
+    end
+  end
+
+  attr_accessor :regenerate_ssh_private_key
+
+  def ssh_key_auth?
+    ssh_mirror_url? && auth_method == 'ssh_public_key'
+  end
+
+  def password_auth?
+    auth_method == 'password'
+  end
+
+  def ssh_mirror_url?
+    url&.start_with?('ssh://')
+  end
+
+  def ssh_known_hosts_verified_by
+    @ssh_known_hosts_verified_by ||= ::User.find_by(id: ssh_known_hosts_verified_by_id)
+  end
+
+  def ssh_known_hosts_fingerprints
+    ::SshHostKey.fingerprint_host_keys(ssh_known_hosts)
+  end
+
+  def auth_method
+    auth_method = credentials.fetch(:auth_method, nil) if credentials.present?
+
+    auth_method.presence || 'password'
+  end
+
+  def ssh_public_key
+    return nil if ssh_private_key.blank?
+
+    comment = "git@#{::Gitlab.config.gitlab.host}"
+    ::SSHKey.new(ssh_private_key, comment: comment).ssh_public_key
+  end
+
+  def generate_ssh_private_key!
+    self.ssh_private_key = ::SSHKey.generate(SSH_PRIVATE_KEY_OPTS).private_key
+  end
+end
diff --git a/app/models/deployment.rb b/app/models/deployment.rb
index 83434276995..811e623b7f7 100644
--- a/app/models/deployment.rb
+++ b/app/models/deployment.rb
@@ -160,18 +160,18 @@ class Deployment < ActiveRecord::Base
   end
 
   def has_metrics?
-    prometheus_adapter&.can_query?
+    prometheus_adapter&.can_query? && success?
   end
 
   def metrics
-    return {} unless has_metrics? && success?
+    return {} unless has_metrics?
 
     metrics = prometheus_adapter.query(:deployment, self)
     metrics&.merge(deployment_time: finished_at.to_i) || {}
   end
 
   def additional_metrics
-    return {} unless has_metrics? && success?
+    return {} unless has_metrics?
 
     metrics = prometheus_adapter.query(:additional_metrics_deployment, self)
     metrics&.merge(deployment_time: finished_at.to_i) || {}
diff --git a/app/models/hooks/service_hook.rb b/app/models/hooks/service_hook.rb
index 7d9f6d89d44..8f305dd7c22 100644
--- a/app/models/hooks/service_hook.rb
+++ b/app/models/hooks/service_hook.rb
@@ -5,8 +5,8 @@ class ServiceHook < WebHook
   validates :service, presence: true
 
   # rubocop: disable CodeReuse/ServiceClass
-  def execute(data)
-    WebHookService.new(self, data, 'service_hook').execute
+  def execute(data, hook_name = 'service_hook')
+    WebHookService.new(self, data, hook_name).execute
   end
   # rubocop: enable CodeReuse/ServiceClass
 end
diff --git a/app/models/identity.rb b/app/models/identity.rb
index f5a13dbd6f2..d63dd432426 100644
--- a/app/models/identity.rb
+++ b/app/models/identity.rb
@@ -1,18 +1,14 @@
 # frozen_string_literal: true
 
 class Identity < ActiveRecord::Base
-  def self.uniqueness_scope
-    :provider
-  end
-
   include Sortable
   include CaseSensitivity
 
   belongs_to :user
 
   validates :provider, presence: true
-  validates :extern_uid, allow_blank: true, uniqueness: { scope: uniqueness_scope, case_sensitive: false }
-  validates :user_id, uniqueness: { scope: uniqueness_scope }
+  validates :extern_uid, allow_blank: true, uniqueness: { scope: UniquenessScopes.scopes, case_sensitive: false }
+  validates :user_id, uniqueness: { scope: UniquenessScopes.scopes }
 
   before_save :ensure_normalized_extern_uid, if: :extern_uid_changed?
   after_destroy :clear_user_synced_attributes, if: :user_synced_attributes_metadata_from_provider?
diff --git a/app/models/identity/uniqueness_scopes.rb b/app/models/identity/uniqueness_scopes.rb
new file mode 100644
index 00000000000..674b735903f
--- /dev/null
+++ b/app/models/identity/uniqueness_scopes.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class Identity < ActiveRecord::Base
+  # This module and method are defined in a separate file to allow EE to
+  # redefine the `scopes` method before it is used in the `Identity` model.
+  module UniquenessScopes
+    def self.scopes
+      [:provider]
+    end
+  end
+end
diff --git a/app/models/member.rb b/app/models/member.rb
index 0696ea46c8b..bc8ac14d148 100644
--- a/app/models/member.rb
+++ b/app/models/member.rb
@@ -152,11 +152,13 @@ class Member < ActiveRecord::Base
 
       return member unless can_update_member?(current_user, member)
 
-      member.attributes = {
-        created_by: member.created_by || current_user,
-        access_level: access_level,
-        expires_at: expires_at
-      }
+      set_member_attributes(
+        member,
+        access_level,
+        current_user: current_user,
+        expires_at: expires_at,
+        ldap: ldap
+      )
 
       if member.request?
         ::Members::ApproveAccessRequestService.new(
@@ -175,6 +177,18 @@ class Member < ActiveRecord::Base
       # rubocop: enable CodeReuse/ServiceClass
     end
 
+    # Populates the attributes of a member.
+    #
+    # This logic resides in a separate method so that EE can extend this logic,
+    # without having to patch the `add_user` method directly.
+    def set_member_attributes(member, access_level, current_user: nil, expires_at: nil, ldap: false)
+      member.attributes = {
+        created_by: member.created_by || current_user,
+        access_level: access_level,
+        expires_at: expires_at
+      }
+    end
+
     def add_users(source, users, access_level, current_user: nil, expires_at: nil)
       return [] unless users.present?
 
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index df5678ec2f1..92add079a02 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -966,7 +966,6 @@ class MergeRequest < ActiveRecord::Base
 
   def mergeable_ci_state?
     return true unless project.only_allow_merge_if_pipeline_succeeds?
-    return true unless head_pipeline
 
     actual_head_pipeline&.success? || actual_head_pipeline&.skipped?
   end
diff --git a/app/models/project.rb b/app/models/project.rb
index e2b65fab3ee..9370e3484b7 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -1903,10 +1903,6 @@ class Project < ActiveRecord::Base
     false
   end
 
-  def issue_board_milestone_available?(user = nil)
-    feature_available?(:issue_board_milestone, user)
-  end
-
   def full_path_was
     File.join(namespace.full_path, previous_changes['path'].first)
   end
@@ -1968,7 +1964,7 @@ class Project < ActiveRecord::Base
   end
 
   def migrate_to_hashed_storage!
-    return if hashed_storage?(:repository)
+    return unless storage_upgradable?
 
     update!(repository_read_only: true)
 
diff --git a/app/models/project_services/chat_message/push_message.rb b/app/models/project_services/chat_message/push_message.rb
index 82be33a12a1..5dd0414b7e6 100644
--- a/app/models/project_services/chat_message/push_message.rb
+++ b/app/models/project_services/chat_message/push_message.rb
@@ -26,16 +26,8 @@ module ChatMessage
     end
 
     def activity
-      action = if new_branch?
-                 "created"
-               elsif removed_branch?
-                 "removed"
-               else
-                 "pushed to"
-               end
-
       {
-        title: "#{user_combined_name} #{action} #{ref_type}",
+        title: humanized_action(short: true),
         subtitle: "in #{project_link}",
         text: compare_link,
         image: user_avatar
@@ -44,32 +36,21 @@ module ChatMessage
 
     private
 
+    def humanized_action(short: false)
+      action, ref_link, target_link = compose_action_details
+      text = [user_combined_name, action, ref_type, ref_link]
+      text << target_link unless short
+      text.join(' ')
+    end
+
     def message
-      if new_branch?
-        new_branch_message
-      elsif removed_branch?
-        removed_branch_message
-      else
-        push_message
-      end
+      humanized_action
     end
 
     def format(string)
       Slack::Notifier::LinkFormatter.format(string)
     end
 
-    def new_branch_message
-      "#{user_combined_name} pushed new #{ref_type} #{branch_link} to #{project_link}"
-    end
-
-    def removed_branch_message
-      "#{user_combined_name} removed #{ref_type} #{ref} from #{project_link}"
-    end
-
-    def push_message
-      "#{user_combined_name} pushed to #{ref_type} #{branch_link} of #{project_link} (#{compare_link})"
-    end
-
     def commit_messages
       commits.map { |commit| compose_commit_message(commit) }.join("\n\n")
     end
@@ -115,6 +96,16 @@ module ChatMessage
       "[Compare changes](#{compare_url})"
     end
 
+    def compose_action_details
+      if new_branch?
+        ['pushed new', branch_link, "to #{project_link}"]
+      elsif removed_branch?
+        ['removed', ref, "from #{project_link}"]
+      else
+        ['pushed to', branch_link, "of #{project_link} (#{compare_link})"]
+      end
+    end
+
     def attachment_color
       '#345'
     end
diff --git a/app/models/project_services/issue_tracker_service.rb b/app/models/project_services/issue_tracker_service.rb
index a399982e5ec..f54497fc6d8 100644
--- a/app/models/project_services/issue_tracker_service.rb
+++ b/app/models/project_services/issue_tracker_service.rb
@@ -9,7 +9,7 @@ class IssueTrackerService < Service
   # Override this method on services that uses different patterns
   # This pattern does not support cross-project references
   # The other code assumes that this pattern is a superset of all
-  # overridden patterns. See ReferenceRegexes::EXTERNAL_PATTERN
+  # overridden patterns. See ReferenceRegexes.external_pattern
   def self.reference_pattern(only_long: false)
     if only_long
       /(\b[A-Z][A-Z0-9_]*-)(?<issue>\d+)/
diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb
index 3459ded7ccf..c52a531e5fe 100644
--- a/app/models/project_services/kubernetes_service.rb
+++ b/app/models/project_services/kubernetes_service.rb
@@ -203,9 +203,7 @@ class KubernetesService < DeploymentService
     kubeclient = build_kube_client!
 
     kubeclient.get_pods(namespace: actual_namespace).as_json
-  rescue Kubeclient::HttpError => err
-    raise err unless err.error_code == 404
-
+  rescue Kubeclient::ResourceNotFoundError
     []
   end
 
diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb
index c1f53b5da4f..a3415a4a14c 100644
--- a/app/models/remote_mirror.rb
+++ b/app/models/remote_mirror.rb
@@ -2,6 +2,7 @@
 
 class RemoteMirror < ActiveRecord::Base
   include AfterCommitQueue
+  include MirrorAuthentication
 
   PROTECTED_BACKOFF_DELAY   = 1.minute
   UNPROTECTED_BACKOFF_DELAY = 5.minutes
@@ -28,6 +29,8 @@ class RemoteMirror < ActiveRecord::Base
 
   after_commit :remove_remote, on: :destroy
 
+  before_validation :store_credentials
+
   scope :enabled, -> { where(enabled: true) }
   scope :started, -> { with_update_status(:started) }
   scope :stuck,   -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) }
@@ -84,7 +87,21 @@ class RemoteMirror < ActiveRecord::Base
   end
 
   def update_repository(options)
-    raw.update(options)
+    if ssh_mirror_url?
+      if ssh_key_auth? && ssh_private_key.present?
+        options[:ssh_key] = ssh_private_key
+      end
+
+      if ssh_known_hosts.present?
+        options[:known_hosts] = ssh_known_hosts
+      end
+    end
+
+    Gitlab::Git::RemoteMirror.new(
+      project.repository.raw,
+      remote_name,
+      **options
+    ).update
   end
 
   def sync?
@@ -128,7 +145,8 @@ class RemoteMirror < ActiveRecord::Base
     super(value) && return unless Gitlab::UrlSanitizer.valid?(value)
 
     mirror_url = Gitlab::UrlSanitizer.new(value)
-    self.credentials = mirror_url.credentials
+    self.credentials ||= {}
+    self.credentials = self.credentials.merge(mirror_url.credentials)
 
     super(mirror_url.sanitized_url)
   end
@@ -152,17 +170,28 @@ class RemoteMirror < ActiveRecord::Base
 
   def ensure_remote!
     return unless project
-    return unless remote_name && url
+    return unless remote_name && remote_url
 
     # If this fails or the remote already exists, we won't know due to
     # https://gitlab.com/gitlab-org/gitaly/issues/1317
-    project.repository.add_remote(remote_name, url)
+    project.repository.add_remote(remote_name, remote_url)
   end
 
   private
 
-  def raw
-    @raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, remote_name)
+  def store_credentials
+    # This is a necessary workaround for attr_encrypted, which doesn't otherwise
+    # notice that the credentials have changed
+    self.credentials = self.credentials
+  end
+
+  # The remote URL omits any password if SSH public-key authentication is in use
+  def remote_url
+    return url unless ssh_key_auth? && password.present?
+
+    Gitlab::UrlSanitizer.new(read_attribute(:url), credentials: { user: user }).full_url
+  rescue
+    super
   end
 
   def fallback_remote_name
@@ -214,7 +243,7 @@ class RemoteMirror < ActiveRecord::Base
       project.repository.async_remove_remote(prev_remote_name)
     end
 
-    project.repository.add_remote(remote_name, url)
+    project.repository.add_remote(remote_name, remote_url)
   end
 
   def remove_remote
@@ -224,6 +253,6 @@ class RemoteMirror < ActiveRecord::Base
   end
 
   def mirror_url_changed?
-    url_changed? || encrypted_credentials_changed?
+    url_changed? || credentials_changed?
   end
 end
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 6e179f61a7b..fff6d4be275 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -1049,11 +1049,19 @@ class Repository
   end
 
   def cache
-    @cache ||= Gitlab::RepositoryCache.new(self)
+    @cache ||= if is_wiki
+                 Gitlab::RepositoryCache.new(self, extra_namespace: 'wiki')
+               else
+                 Gitlab::RepositoryCache.new(self)
+               end
   end
 
   def request_store_cache
-    @request_store_cache ||= Gitlab::RepositoryCache.new(self, backend: Gitlab::SafeRequestStore)
+    @request_store_cache ||= if is_wiki
+                               Gitlab::RepositoryCache.new(self, extra_namespace: 'wiki', backend: Gitlab::SafeRequestStore)
+                             else
+                               Gitlab::RepositoryCache.new(self, backend: Gitlab::SafeRequestStore)
+                             end
   end
 
   def tags_sorted_by_committed_date
diff --git a/app/models/shard.rb b/app/models/shard.rb
index 2fa22bd040c..2e75bc91df0 100644
--- a/app/models/shard.rb
+++ b/app/models/shard.rb
@@ -9,13 +9,12 @@ class Shard < ActiveRecord::Base
 
     # The GitLab config does not change for the lifecycle of the process
     in_config = Gitlab.config.repositories.storages.keys.map(&:to_s)
+    in_db = all.pluck(:name)
 
-    transaction do
-      in_db = all.pluck(:name)
-      missing = in_config - in_db
-
-      missing.map { |name| by_name(name) }
-    end
+    # This may race with other processes creating shards at the same time, but
+    # `by_name` will handle that correctly
+    missing = in_config - in_db
+    missing.map { |name| by_name(name) }
   end
 
   def self.by_name(name)
diff --git a/app/models/user.rb b/app/models/user.rb
index a400058e87e..01eba7e0426 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -349,20 +349,28 @@ class User < ActiveRecord::Base
     def find_by_any_email(email, confirmed: false)
       return unless email
 
-      downcased = email.downcase
-
-      find_by_private_commit_email(downcased) || by_any_email(downcased, confirmed: confirmed).take
+      by_any_email(email, confirmed: confirmed).take
     end
 
-    # Returns a relation containing all the users for the given Email address
-    def by_any_email(email, confirmed: false)
-      users = where(email: email)
-      users = users.confirmed if confirmed
+    # Returns a relation containing all the users for the given email addresses
+    #
+    # @param emails [String, Array<String>] email addresses to check
+    # @param confirmed [Boolean] Only return users where the email is confirmed
+    def by_any_email(emails, confirmed: false)
+      emails = Array(emails).map(&:downcase)
+
+      from_users = where(email: emails)
+      from_users = from_users.confirmed if confirmed
 
-      emails = joins(:emails).where(emails: { email: email })
-      emails = emails.confirmed if confirmed
+      from_emails = joins(:emails).where(emails: { email: emails })
+      from_emails = from_emails.confirmed if confirmed
 
-      from_union([users, emails])
+      items = [from_users, from_emails]
+
+      user_ids = Gitlab::PrivateCommitEmail.user_ids_for_emails(emails)
+      items << where(id: user_ids) if user_ids.present?
+
+      from_union(items)
     end
 
     def find_by_private_commit_email(email)
@@ -1031,6 +1039,7 @@ class User < ActiveRecord::Base
   def all_emails
     all_emails = []
     all_emails << email unless temp_oauth_email?
+    all_emails << private_commit_email
     all_emails.concat(emails.map(&:email))
     all_emails
   end
@@ -1043,16 +1052,24 @@ class User < ActiveRecord::Base
     verified_emails
   end
 
+  def any_email?(check_email)
+    downcased = check_email.downcase
+
+    # handle the outdated private commit email case
+    return true if persisted? &&
+        id == Gitlab::PrivateCommitEmail.user_id_for_email(downcased)
+
+    all_emails.include?(check_email.downcase)
+  end
+
   def verified_email?(check_email)
     downcased = check_email.downcase
 
-    if email == downcased
-      primary_email_verified?
-    else
-      user_id = Gitlab::PrivateCommitEmail.user_id_for_email(downcased)
+    # handle the outdated private commit email case
+    return true if persisted? &&
+        id == Gitlab::PrivateCommitEmail.user_id_for_email(downcased)
 
-      user_id == id || emails.confirmed.where(email: downcased).exists?
-    end
+    verified_emails.include?(check_email.downcase)
   end
 
   def hook_attrs
diff --git a/app/models/user_callout.rb b/app/models/user_callout.rb
index 1cd05cf3aac..76e7bc06b4e 100644
--- a/app/models/user_callout.rb
+++ b/app/models/user_callout.rb
@@ -3,11 +3,9 @@
 class UserCallout < ActiveRecord::Base
   belongs_to :user
 
-  enum feature_name: {
-    gke_cluster_integration: 1,
-    gcp_signup_offer: 2,
-    cluster_security_warning: 3
-  }
+  # We use `UserCalloutEnums.feature_names` here so that EE can more easily
+  # extend this `Hash` with new values.
+  enum feature_name: ::UserCalloutEnums.feature_names
 
   validates :user, presence: true
   validates :feature_name,
diff --git a/app/models/user_callout_enums.rb b/app/models/user_callout_enums.rb
new file mode 100644
index 00000000000..b9373ae6166
--- /dev/null
+++ b/app/models/user_callout_enums.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module UserCalloutEnums
+  # Returns the `Hash` to use for the `feature_name` enum in the `UserCallout`
+  # model.
+  #
+  # This method is separate from the `UserCallout` model so that it can be
+  # extended by EE.
+  def self.feature_names
+    {
+      gke_cluster_integration: 1,
+      gcp_signup_offer: 2,
+      cluster_security_warning: 3
+    }
+  end
+end