Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (507 commits)
  Add dropdowns documentation
  Convert migration to populate latest merge request ID into a background migration
  Set 0.69.0 instead of latest for codeclimate image
  De-duplicate background migration matchers defined in spec/support/migrations_helpers.rb
  Update database_debugging.md
  Update database_debugging.md
  Move installation of apps higher
  Change to Google Kubernetes Cluster and add internal links
  Add Ingress description from official docs
  Add info on creating your own k8s cluster from the cluster page
  Add info about the installed apps in the Cluster docs
  Resolve "lock/confidential issuable sidebar custom svg icons iteration"
  Update HA README.md to clarify GitLab support does not troubleshoot DRBD.
  Update license_finder to 3.1.1
  Make sure NotesActions#noteable returns a Noteable in the update action
  Cache the number of user SSH keys
  Adjust openid_connect_spec to use `raise_error`
  Resolve "Clicking on GPG verification badge jumps to top of the page"
  Add changelog for container repository path update
  Update container repository path reference
  ...

33 files changed, 643 insertions, 213 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 6ca46ae89c1..1b2b0d17910 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -192,6 +192,10 @@ module Ci
       project.build_timeout
     end
 
+    def triggered_by?(current_user)
+      user == current_user
+    end
+
     # A slugified version of the build ref, suitable for inclusion in URLs and
     # domain names. Rules:
     #
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index ca65e81f27a..19814864e50 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -66,8 +66,8 @@ module Ci
 
     state_machine :status, initial: :created do
       event :enqueue do
-        transition created: :pending
-        transition [:success, :failed, :canceled, :skipped] => :running
+        transition [:created, :skipped] => :pending
+        transition [:success, :failed, :canceled] => :running
       end
 
       event :run do
@@ -409,7 +409,7 @@ module Ci
     end
 
     def notes
-      Note.for_commit_id(sha)
+      project.notes.for_commit_id(sha)
     end
 
     def process!
diff --git a/app/models/clusters/applications/helm.rb b/app/models/clusters/applications/helm.rb
new file mode 100644
index 00000000000..c7949d11ef8
--- /dev/null
+++ b/app/models/clusters/applications/helm.rb
@@ -0,0 +1,35 @@
+module Clusters
+  module Applications
+    class Helm < ActiveRecord::Base
+      self.table_name = 'clusters_applications_helm'
+
+      include ::Clusters::Concerns::ApplicationStatus
+
+      belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id
+
+      default_value_for :version, Gitlab::Kubernetes::Helm::HELM_VERSION
+
+      validates :cluster, presence: true
+
+      after_initialize :set_initial_status
+
+      def self.application_name
+        self.to_s.demodulize.underscore
+      end
+
+      def set_initial_status
+        return unless not_installable?
+
+        self.status = 'installable' if cluster&.platform_kubernetes_active?
+      end
+
+      def name
+        self.class.application_name
+      end
+
+      def install_command
+        Gitlab::Kubernetes::Helm::InstallCommand.new(name, true)
+      end
+    end
+  end
+end
diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb
new file mode 100644
index 00000000000..44bd979741e
--- /dev/null
+++ b/app/models/clusters/applications/ingress.rb
@@ -0,0 +1,44 @@
+module Clusters
+  module Applications
+    class Ingress < ActiveRecord::Base
+      self.table_name = 'clusters_applications_ingress'
+
+      include ::Clusters::Concerns::ApplicationStatus
+
+      belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id
+
+      validates :cluster, presence: true
+
+      default_value_for :ingress_type, :nginx
+      default_value_for :version, :nginx
+
+      after_initialize :set_initial_status
+
+      enum ingress_type: {
+        nginx: 1
+      }
+
+      def self.application_name
+        self.to_s.demodulize.underscore
+      end
+
+      def set_initial_status
+        return unless not_installable?
+
+        self.status = 'installable' if cluster&.application_helm_installed?
+      end
+
+      def name
+        self.class.application_name
+      end
+
+      def chart
+        'stable/nginx-ingress'
+      end
+
+      def install_command
+        Gitlab::Kubernetes::Helm::InstallCommand.new(name, false, chart)
+      end
+    end
+  end
+end
diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb
new file mode 100644
index 00000000000..185d9473aab
--- /dev/null
+++ b/app/models/clusters/cluster.rb
@@ -0,0 +1,102 @@
+module Clusters
+  class Cluster < ActiveRecord::Base
+    include Presentable
+
+    self.table_name = 'clusters'
+
+    APPLICATIONS = {
+      Applications::Helm.application_name => Applications::Helm,
+      Applications::Ingress.application_name => Applications::Ingress
+    }.freeze
+
+    belongs_to :user
+
+    has_many :cluster_projects, class_name: 'Clusters::Project'
+    has_many :projects, through: :cluster_projects, class_name: '::Project'
+
+    # we force autosave to happen when we save `Cluster` model
+    has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true
+
+    # We have to ":destroy" it today to ensure that we clean also the Kubernetes Integration
+    has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+
+    has_one :application_helm, class_name: 'Clusters::Applications::Helm'
+    has_one :application_ingress, class_name: 'Clusters::Applications::Ingress'
+
+    accepts_nested_attributes_for :provider_gcp, update_only: true
+    accepts_nested_attributes_for :platform_kubernetes, update_only: true
+
+    validates :name, cluster_name: true
+    validate :restrict_modification, on: :update
+
+    # TODO: Move back this into Clusters::Platforms::Kubernetes in 10.3
+    # We need callback here because `enabled` belongs to Clusters::Cluster
+    # Callbacks in Clusters::Platforms::Kubernetes will not be called after update
+    after_save :update_kubernetes_integration!
+
+    delegate :status, to: :provider, allow_nil: true
+    delegate :status_reason, to: :provider, allow_nil: true
+    delegate :on_creation?, to: :provider, allow_nil: true
+    delegate :update_kubernetes_integration!, to: :platform, allow_nil: true
+
+    delegate :active?, to: :platform_kubernetes, prefix: true, allow_nil: true
+    delegate :installed?, to: :application_helm, prefix: true, allow_nil: true
+
+    enum platform_type: {
+      kubernetes: 1
+    }
+
+    enum provider_type: {
+      user: 0,
+      gcp: 1
+    }
+
+    scope :enabled, -> { where(enabled: true) }
+    scope :disabled, -> { where(enabled: false) }
+
+    def status_name
+      if provider
+        provider.status_name
+      else
+        :created
+      end
+    end
+
+    def applications
+      [
+        application_helm || build_application_helm,
+        application_ingress || build_application_ingress
+      ]
+    end
+
+    def provider
+      return provider_gcp if gcp?
+    end
+
+    def platform
+      return platform_kubernetes if kubernetes?
+    end
+
+    def first_project
+      return @first_project if defined?(@first_project)
+
+      @first_project = projects.first
+    end
+    alias_method :project, :first_project
+
+    def kubeclient
+      platform_kubernetes.kubeclient if kubernetes?
+    end
+
+    private
+
+    def restrict_modification
+      if provider&.on_creation?
+        errors.add(:base, "cannot modify during creation")
+        return false
+      end
+
+      true
+    end
+  end
+end
diff --git a/app/models/clusters/concerns/application_status.rb b/app/models/clusters/concerns/application_status.rb
new file mode 100644
index 00000000000..7b7c8eac773
--- /dev/null
+++ b/app/models/clusters/concerns/application_status.rb
@@ -0,0 +1,43 @@
+module Clusters
+  module Concerns
+    module ApplicationStatus
+      extend ActiveSupport::Concern
+
+      included do
+        state_machine :status, initial: :not_installable do
+          state :not_installable, value: -2
+          state :errored, value: -1
+          state :installable, value: 0
+          state :scheduled, value: 1
+          state :installing, value: 2
+          state :installed, value: 3
+
+          event :make_scheduled do
+            transition [:installable, :errored] => :scheduled
+          end
+
+          event :make_installing do
+            transition [:scheduled] => :installing
+          end
+
+          event :make_installed do
+            transition [:installing] => :installed
+          end
+
+          event :make_errored do
+            transition any => :errored
+          end
+
+          before_transition any => [:scheduled] do |app_status, _|
+            app_status.status_reason = nil
+          end
+
+          before_transition any => [:errored] do |app_status, transition|
+            status_reason = transition.args.first
+            app_status.status_reason = status_reason if status_reason
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb
new file mode 100644
index 00000000000..6dc1ee810d3
--- /dev/null
+++ b/app/models/clusters/platforms/kubernetes.rb
@@ -0,0 +1,109 @@
+module Clusters
+  module Platforms
+    class Kubernetes < ActiveRecord::Base
+      self.table_name = 'cluster_platforms_kubernetes'
+
+      belongs_to :cluster, inverse_of: :platform_kubernetes, class_name: 'Clusters::Cluster'
+
+      attr_encrypted :password,
+        mode: :per_attribute_iv,
+        key: Gitlab::Application.secrets.db_key_base,
+        algorithm: 'aes-256-cbc'
+
+      attr_encrypted :token,
+        mode: :per_attribute_iv,
+        key: Gitlab::Application.secrets.db_key_base,
+        algorithm: 'aes-256-cbc'
+
+      before_validation :enforce_namespace_to_lower_case
+
+      validates :namespace,
+        allow_blank: true,
+        length: 1..63,
+        format: {
+          with: Gitlab::Regex.kubernetes_namespace_regex,
+          message: Gitlab::Regex.kubernetes_namespace_regex_message
+        }
+
+      # We expect to be `active?` only when enabled and cluster is created (the api_url is assigned)
+      validates :api_url, url: true, presence: true
+      validates :token, presence: true
+
+      # TODO: Glue code till we migrate Kubernetes Integration into Platforms::Kubernetes
+      after_destroy :destroy_kubernetes_integration!
+
+      alias_attribute :ca_pem, :ca_cert
+
+      delegate :project, to: :cluster, allow_nil: true
+      delegate :enabled?, to: :cluster, allow_nil: true
+
+      class << self
+        def namespace_for_project(project)
+          "#{project.path}-#{project.id}"
+        end
+      end
+
+      def actual_namespace
+        if namespace.present?
+          namespace
+        else
+          default_namespace
+        end
+      end
+
+      def default_namespace
+        self.class.namespace_for_project(project) if project
+      end
+
+      def kubeclient
+        @kubeclient ||= kubernetes_service.kubeclient if manages_kubernetes_service?
+      end
+
+      def update_kubernetes_integration!
+        raise 'Kubernetes service already configured' unless manages_kubernetes_service?
+
+        # This is neccesary, otheriwse enabled? returns true even though cluster updated with enabled: false
+        cluster.reload
+
+        ensure_kubernetes_service&.update!(
+          active: enabled?,
+          api_url: api_url,
+          namespace: namespace,
+          token: token,
+          ca_pem: ca_cert
+        )
+      end
+
+      def active?
+        manages_kubernetes_service?
+      end
+
+      private
+
+      def enforce_namespace_to_lower_case
+        self.namespace = self.namespace&.downcase
+      end
+
+      # TODO: glue code till we migrate Kubernetes Service into Platforms::Kubernetes class
+      def manages_kubernetes_service?
+        return true unless kubernetes_service&.active?
+
+        kubernetes_service.api_url == api_url
+      end
+
+      def destroy_kubernetes_integration!
+        return unless manages_kubernetes_service?
+
+        kubernetes_service&.destroy!
+      end
+
+      def kubernetes_service
+        @kubernetes_service ||= project&.kubernetes_service
+      end
+
+      def ensure_kubernetes_service
+        @kubernetes_service ||= kubernetes_service || project&.build_kubernetes_service
+      end
+    end
+  end
+end
diff --git a/app/models/clusters/project.rb b/app/models/clusters/project.rb
new file mode 100644
index 00000000000..eeb734b20b8
--- /dev/null
+++ b/app/models/clusters/project.rb
@@ -0,0 +1,8 @@
+module Clusters
+  class Project < ActiveRecord::Base
+    self.table_name = 'cluster_projects'
+
+    belongs_to :cluster, class_name: 'Clusters::Cluster'
+    belongs_to :project, class_name: '::Project'
+  end
+end
diff --git a/app/models/clusters/providers/gcp.rb b/app/models/clusters/providers/gcp.rb
new file mode 100644
index 00000000000..ee2e43ee9dd
--- /dev/null
+++ b/app/models/clusters/providers/gcp.rb
@@ -0,0 +1,79 @@
+module Clusters
+  module Providers
+    class Gcp < ActiveRecord::Base
+      self.table_name = 'cluster_providers_gcp'
+
+      belongs_to :cluster, inverse_of: :provider_gcp, class_name: 'Clusters::Cluster'
+
+      default_value_for :zone, 'us-central1-a'
+      default_value_for :num_nodes, 3
+      default_value_for :machine_type, 'n1-standard-2'
+
+      attr_encrypted :access_token,
+        mode: :per_attribute_iv,
+        key: Gitlab::Application.secrets.db_key_base,
+        algorithm: 'aes-256-cbc'
+
+      validates :gcp_project_id,
+        length: 1..63,
+        format: {
+          with: Gitlab::Regex.kubernetes_namespace_regex,
+          message: Gitlab::Regex.kubernetes_namespace_regex_message
+        }
+
+      validates :zone, presence: true
+
+      validates :num_nodes,
+        presence: true,
+        numericality: {
+          only_integer: true,
+          greater_than: 0
+        }
+
+      state_machine :status, initial: :scheduled do
+        state :scheduled, value: 1
+        state :creating, value: 2
+        state :created, value: 3
+        state :errored, value: 4
+
+        event :make_creating do
+          transition any - [:creating] => :creating
+        end
+
+        event :make_created do
+          transition any - [:created] => :created
+        end
+
+        event :make_errored do
+          transition any - [:errored] => :errored
+        end
+
+        before_transition any => [:errored, :created] do |provider|
+          provider.access_token = nil
+          provider.operation_id = nil
+        end
+
+        before_transition any => [:creating] do |provider, transition|
+          operation_id = transition.args.first
+          raise ArgumentError.new('operation_id is required') unless operation_id.present?
+          provider.operation_id = operation_id
+        end
+
+        before_transition any => [:errored] do |provider, transition|
+          status_reason = transition.args.first
+          provider.status_reason = status_reason if status_reason
+        end
+      end
+
+      def on_creation?
+        scheduled? || creating?
+      end
+
+      def api_client
+        return unless access_token
+
+        @api_client ||= GoogleApi::CloudPlatform::Client.new(access_token, nil)
+      end
+    end
+  end
+end
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index f3888528940..6b07dbdf3ea 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -14,7 +14,6 @@ class CommitStatus < ActiveRecord::Base
   delegate :sha, :short_sha, to: :pipeline
 
   validates :pipeline, presence: true, unless: :importing?
-
   validates :name, presence: true, unless: :importing?
 
   alias_attribute :author, :user
@@ -46,6 +45,17 @@ class CommitStatus < ActiveRecord::Base
     runner_system_failure: 4
   }
 
+  ##
+  # We still create some CommitStatuses outside of CreatePipelineService.
+  #
+  # These are pages deployments and external statuses.
+  #
+  before_create unless: :importing? do
+    Ci::EnsureStageService.new(project, user).execute(self) do |stage|
+      self.run_after_commit { StageUpdateWorker.perform_async(stage.id) }
+    end
+  end
+
   state_machine :status do
     event :process do
       transition [:skipped, :manual] => :created
diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb
index 2ec70203710..10659030910 100644
--- a/app/models/concerns/avatarable.rb
+++ b/app/models/concerns/avatarable.rb
@@ -4,15 +4,26 @@ module Avatarable
   def avatar_path(only_path: true)
     return unless self[:avatar].present?
 
-    # If only_path is true then use the relative path of avatar.
-    # Otherwise use full path (including host).
     asset_host = ActionController::Base.asset_host
-    gitlab_host = only_path ? gitlab_config.relative_url_root : gitlab_config.url
+    use_asset_host = asset_host.present?
 
-    # If asset_host is set then it is expected that assets are handled by a standalone host.
-    # That means we do not want to get GitLab's relative_url_root option anymore.
-    host = (asset_host.present? && (!respond_to?(:public?) || public?)) ? asset_host : gitlab_host
+    # Avatars for private and internal groups and projects require authentication to be viewed,
+    # which means they can only be served by Rails, on the regular GitLab host.
+    # If an asset host is configured, we need to return the fully qualified URL
+    # instead of only the avatar path, so that Rails doesn't prefix it with the asset host.
+    if use_asset_host && respond_to?(:public?) && !public?
+      use_asset_host = false
+      only_path = false
+    end
 
-    [host, avatar.url].join
+    url_base = ""
+    if use_asset_host
+      url_base << asset_host unless only_path
+    else
+      url_base << gitlab_config.base_url unless only_path
+      url_base << gitlab_config.relative_url_root
+    end
+
+    url_base + avatar.url
   end
 end
diff --git a/app/models/concerns/ignorable_column.rb b/app/models/concerns/ignorable_column.rb
index eb9f3423e48..03793e8bcbb 100644
--- a/app/models/concerns/ignorable_column.rb
+++ b/app/models/concerns/ignorable_column.rb
@@ -21,8 +21,8 @@ module IgnorableColumn
       @ignored_columns ||= Set.new
     end
 
-    def ignore_column(name)
-      ignored_columns << name.to_s
+    def ignore_column(*names)
+      ignored_columns.merge(names.map(&:to_s))
     end
   end
 end
diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb
index a928b9d6367..35090181bd9 100644
--- a/app/models/concerns/issuable.rb
+++ b/app/models/concerns/issuable.rb
@@ -17,6 +17,8 @@ module Issuable
   include Importable
   include Editable
   include AfterCommitQueue
+  include Sortable
+  include CreatedAtFilterable
 
   # This object is used to gather issuable meta data for displaying
   # upvotes, downvotes, notes and closing merge requests count for issues and merge requests
@@ -253,7 +255,7 @@ module Issuable
     participants(user).include?(user)
   end
 
-  def to_hook_data(user, old_labels: [], old_assignees: [])
+  def to_hook_data(user, old_labels: [], old_assignees: [], old_total_time_spent: nil)
     changes = previous_changes
 
     if old_labels != labels
@@ -268,6 +270,10 @@ module Issuable
       end
     end
 
+    if old_total_time_spent != total_time_spent
+      changes[:total_time_spent] = [old_total_time_spent, total_time_spent]
+    end
+
     Gitlab::HookData::IssuableBuilder.new(self).build(user: user, changes: changes)
   end
 
diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb
index 0f506e6aa25..c22fb01a4ba 100644
--- a/app/models/concerns/milestoneish.rb
+++ b/app/models/concerns/milestoneish.rb
@@ -86,6 +86,14 @@ module Milestoneish
     false
   end
 
+  def total_issue_time_spent
+    @total_issue_time_spent ||= issues.joins(:timelogs).sum(:time_spent)
+  end
+
+  def human_total_issue_time_spent
+    Gitlab::TimeTrackingFormatter.output(total_issue_time_spent)
+  end
+
   private
 
   def count_issues_by_state(user)
diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb
index d88a92dc027..ae5f138a920 100644
--- a/app/models/diff_note.rb
+++ b/app/models/diff_note.rb
@@ -18,7 +18,8 @@ class DiffNote < Note
   validate :positions_complete
   validate :verify_supported
 
-  before_validation :set_original_position, :update_position, on: :create
+  before_validation :set_original_position, on: :create
+  before_validation :update_position, on: :create, if: :on_text?
   before_validation :set_line_code
   after_save :keep_around_commits
 
diff --git a/app/models/external_issue.rb b/app/models/external_issue.rb
index 0bf18e529f0..9ff56f229bc 100644
--- a/app/models/external_issue.rb
+++ b/app/models/external_issue.rb
@@ -47,4 +47,8 @@ class ExternalIssue
 
     id
   end
+
+  def notes
+    Note.none
+  end
 end
diff --git a/app/models/gcp/cluster.rb b/app/models/gcp/cluster.rb
deleted file mode 100644
index 162a690c0e3..00000000000
--- a/app/models/gcp/cluster.rb
+++ /dev/null
@@ -1,116 +0,0 @@
-module Gcp
-  class Cluster < ActiveRecord::Base
-    extend Gitlab::Gcp::Model
-    include Presentable
-
-    belongs_to :project, inverse_of: :cluster
-    belongs_to :user
-    belongs_to :service
-
-    scope :enabled, -> { where(enabled: true) }
-    scope :disabled, -> { where(enabled: false) }
-
-    default_value_for :gcp_cluster_zone, 'us-central1-a'
-    default_value_for :gcp_cluster_size, 3
-    default_value_for :gcp_machine_type, 'n1-standard-4'
-
-    attr_encrypted :password,
-      mode: :per_attribute_iv,
-      key: Gitlab::Application.secrets.db_key_base,
-      algorithm: 'aes-256-cbc'
-
-    attr_encrypted :kubernetes_token,
-      mode: :per_attribute_iv,
-      key: Gitlab::Application.secrets.db_key_base,
-      algorithm: 'aes-256-cbc'
-
-    attr_encrypted :gcp_token,
-      mode: :per_attribute_iv,
-      key: Gitlab::Application.secrets.db_key_base,
-      algorithm: 'aes-256-cbc'
-
-    validates :gcp_project_id,
-      length: 1..63,
-      format: {
-        with: Gitlab::Regex.kubernetes_namespace_regex,
-        message: Gitlab::Regex.kubernetes_namespace_regex_message
-      }
-
-    validates :gcp_cluster_name,
-      length: 1..63,
-      format: {
-        with: Gitlab::Regex.kubernetes_namespace_regex,
-        message: Gitlab::Regex.kubernetes_namespace_regex_message
-      }
-
-    validates :gcp_cluster_zone, presence: true
-
-    validates :gcp_cluster_size,
-      presence: true,
-      numericality: {
-        only_integer: true,
-        greater_than: 0
-      }
-
-    validates :project_namespace,
-      allow_blank: true,
-      length: 1..63,
-      format: {
-        with: Gitlab::Regex.kubernetes_namespace_regex,
-        message: Gitlab::Regex.kubernetes_namespace_regex_message
-      }
-
-    # if we do not do status transition we prevent change
-    validate :restrict_modification, on: :update, unless: :status_changed?
-
-    state_machine :status, initial: :scheduled do
-      state :scheduled, value: 1
-      state :creating, value: 2
-      state :created, value: 3
-      state :errored, value: 4
-
-      event :make_creating do
-        transition any - [:creating] => :creating
-      end
-
-      event :make_created do
-        transition any - [:created] => :created
-      end
-
-      event :make_errored do
-        transition any - [:errored] => :errored
-      end
-
-      before_transition any => [:errored, :created] do |cluster|
-        cluster.gcp_token = nil
-        cluster.gcp_operation_id = nil
-      end
-
-      before_transition any => [:errored] do |cluster, transition|
-        status_reason = transition.args.first
-        cluster.status_reason = status_reason if status_reason
-      end
-    end
-
-    def project_namespace_placeholder
-      "#{project.path}-#{project.id}"
-    end
-
-    def on_creation?
-      scheduled? || creating?
-    end
-
-    def api_url
-      'https://' + endpoint if endpoint
-    end
-
-    def restrict_modification
-      if on_creation?
-        errors.add(:base, "cannot modify during creation")
-        return false
-      end
-
-      true
-    end
-  end
-end
diff --git a/app/models/group.rb b/app/models/group.rb
index c660de7fcb6..8cf632fb566 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -26,6 +26,7 @@ class Group < Namespace
   has_many :notification_settings, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent
   has_many :labels, class_name: 'GroupLabel'
   has_many :variables, class_name: 'Ci::GroupVariable'
+  has_many :custom_attributes, class_name: 'GroupCustomAttribute'
 
   validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? }
   validate :visibility_level_allowed_by_projects
diff --git a/app/models/group_custom_attribute.rb b/app/models/group_custom_attribute.rb
new file mode 100644
index 00000000000..8157d602d67
--- /dev/null
+++ b/app/models/group_custom_attribute.rb
@@ -0,0 +1,6 @@
+class GroupCustomAttribute < ActiveRecord::Base
+  belongs_to :group
+
+  validates :group, :key, :value, presence: true
+  validates :key, uniqueness: { scope: [:group_id] }
+end
diff --git a/app/models/issue.rb b/app/models/issue.rb
index fc590f9257e..b5abc8f57b0 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -5,11 +5,9 @@ class Issue < ActiveRecord::Base
   include Issuable
   include Noteable
   include Referable
-  include Sortable
   include Spammable
   include FasterCacheKeys
   include RelativePositioning
-  include CreatedAtFilterable
   include TimeTrackable
 
   DueDateStruct = Struct.new(:title, :name).freeze
@@ -264,10 +262,6 @@ class Issue < ActiveRecord::Base
     true
   end
 
-  def update_project_counter_caches?
-    state_changed? || confidential_changed?
-  end
-
   def update_project_counter_caches
     Projects::OpenIssuesCountService.new(project).refresh_cache
   end
diff --git a/app/models/key.rb b/app/models/key.rb
index f119b15c737..815fd1de909 100644
--- a/app/models/key.rb
+++ b/app/models/key.rb
@@ -27,8 +27,10 @@ class Key < ActiveRecord::Base
 
   after_commit :add_to_shell, on: :create
   after_create :post_create_hook
+  after_create :refresh_user_cache
   after_commit :remove_from_shell, on: :destroy
   after_destroy :post_destroy_hook
+  after_destroy :refresh_user_cache
 
   def key=(value)
     value&.delete!("\n\r")
@@ -76,6 +78,12 @@ class Key < ActiveRecord::Base
     )
   end
 
+  def refresh_user_cache
+    return unless user
+
+    Users::KeysCountService.new(user).refresh_cache
+  end
+
   def post_destroy_hook
     SystemHooksService.new.execute_hooks_for(self, :destroy)
   end
diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb
index b7cf96abe83..fc586fa216e 100644
--- a/app/models/lfs_object.rb
+++ b/app/models/lfs_object.rb
@@ -6,16 +6,8 @@ class LfsObject < ActiveRecord::Base
 
   mount_uploader :file, LfsObjectUploader
 
-  def storage_project(project)
-    if project && project.forked?
-      storage_project(project.forked_from_project)
-    else
-      project
-    end
-  end
-
   def project_allowed_access?(project)
-    projects.exists?(storage_project(project).id)
+    projects.exists?(project.lfs_storage_project.id)
   end
 
   def self.destroy_unreferenced
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 3133dc9e7eb..f1a5cc73e83 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -3,12 +3,11 @@ class MergeRequest < ActiveRecord::Base
   include Issuable
   include Noteable
   include Referable
-  include Sortable
   include IgnorableColumn
-  include CreatedAtFilterable
   include TimeTrackable
 
-  ignore_column :locked_at
+  ignore_column :locked_at,
+                :ref_fetched
 
   belongs_to :target_project, class_name: "Project"
   belongs_to :source_project, class_name: "Project"
@@ -426,7 +425,7 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def create_merge_request_diff
-    fetch_ref
+    fetch_ref!
 
     # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37435
     Gitlab::GitalyClient.allow_n_plus_1_calls do
@@ -577,7 +576,7 @@ class MergeRequest < ActiveRecord::Base
     commit_notes = Note
       .except(:order)
       .where(project_id: [source_project_id, target_project_id])
-      .where(noteable_type: 'Commit', commit_id: commit_ids)
+      .for_commit_id(commit_ids)
 
     # We're using a UNION ALL here since this results in better performance
     # compared to using OR statements. We're using UNION ALL since the queries
@@ -811,29 +810,14 @@ class MergeRequest < ActiveRecord::Base
     end
   end
 
-  def fetch_ref
-    write_ref
-    update_column(:ref_fetched, true)
+  def fetch_ref!
+    target_project.repository.fetch_source_branch!(source_project.repository, source_branch, ref_path)
   end
 
   def ref_path
     "refs/#{Repository::REF_MERGE_REQUEST}/#{iid}/head"
   end
 
-  def ref_fetched?
-    super ||
-      begin
-        computed_value = project.repository.ref_exists?(ref_path)
-        update_column(:ref_fetched, true) if computed_value
-
-        computed_value
-      end
-  end
-
-  def ensure_ref_fetched
-    fetch_ref unless ref_fetched?
-  end
-
   def in_locked_state
     begin
       lock_mr
@@ -881,7 +865,19 @@ class MergeRequest < ActiveRecord::Base
   #
   def all_commit_shas
     if persisted?
-      column_shas = MergeRequestDiffCommit.where(merge_request_diff: merge_request_diffs).limit(10_000).pluck('sha')
+      # MySQL doesn't support LIMIT in a subquery.
+      diffs_relation =
+        if Gitlab::Database.postgresql?
+          merge_request_diffs.order(id: :desc).limit(100)
+        else
+          merge_request_diffs
+        end
+
+      column_shas = MergeRequestDiffCommit
+                      .where(merge_request_diff: diffs_relation)
+                      .limit(10_000)
+                      .pluck('sha')
+
       serialised_shas = merge_request_diffs.where.not(st_commits: nil).flat_map(&:commit_shas)
 
       (column_shas + serialised_shas).uniq
@@ -962,10 +958,6 @@ class MergeRequest < ActiveRecord::Base
     true
   end
 
-  def update_project_counter_caches?
-    state_changed?
-  end
-
   def update_project_counter_caches
     Projects::OpenMergeRequestsCountService.new(target_project).refresh_cache
   end
@@ -975,10 +967,4 @@ class MergeRequest < ActiveRecord::Base
 
     project.merge_requests.merged.where(author_id: author_id).empty?
   end
-
-  private
-
-  def write_ref
-    target_project.repository.fetch_source_branch(source_project.repository, source_branch, ref_path)
-  end
 end
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index 0601a61a926..4d401e7ba18 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -36,7 +36,7 @@ class Namespace < ActiveRecord::Base
   validates :path,
     presence: true,
     length: { maximum: 255 },
-    dynamic_path: true
+    namespace_path: true
 
   validate :nesting_level_allowed
 
diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb
index 2e824cda525..43c77f3f2a2 100644
--- a/app/models/pages_domain.rb
+++ b/app/models/pages_domain.rb
@@ -69,6 +69,10 @@ class PagesDomain < ActiveRecord::Base
     current < x509.not_before || x509.not_after < current
   end
 
+  def expiration
+    x509&.not_after
+  end
+
   def subject
     return unless x509
     x509.subject.to_s
diff --git a/app/models/project.rb b/app/models/project.rb
index 3f810ee977b..894ded2a9f6 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -186,7 +186,10 @@ class Project < ActiveRecord::Base
   has_one :import_data, class_name: 'ProjectImportData', inverse_of: :project, autosave: true
   has_one :project_feature, inverse_of: :project
   has_one :statistics, class_name: 'ProjectStatistics'
-  has_one :cluster, class_name: 'Gcp::Cluster', inverse_of: :project
+
+  has_one :cluster_project, class_name: 'Clusters::Project'
+  has_one :cluster, through: :cluster_project, class_name: 'Clusters::Cluster'
+  has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster'
 
   # Container repositories need to remove data from the container registry,
   # which is not managed by the DB. Hence we're still using dependent: :destroy
@@ -213,6 +216,7 @@ class Project < ActiveRecord::Base
   has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
 
   has_one :auto_devops, class_name: 'ProjectAutoDevops'
+  has_many :custom_attributes, class_name: 'ProjectCustomAttribute'
 
   accepts_nested_attributes_for :variables, allow_destroy: true
   accepts_nested_attributes_for :project_feature, update_only: true
@@ -240,10 +244,8 @@ class Project < ActiveRecord::Base
               message: Gitlab::Regex.project_name_regex_message }
   validates :path,
     presence: true,
-    dynamic_path: true,
+    project_path: true,
     length: { maximum: 255 },
-    format: { with: Gitlab::PathRegex.project_path_format_regex,
-              message: Gitlab::PathRegex.project_path_format_message },
     uniqueness: { scope: :namespace_id }
 
   validates :namespace, presence: true
@@ -363,6 +365,7 @@ class Project < ActiveRecord::Base
   scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) }
 
   scope :excluding_project, ->(project) { where.not(id: project) }
+  scope :import_started, -> { where(import_status: 'started') }
 
   state_machine :import_status, initial: :none do
     event :import_schedule do
@@ -701,10 +704,6 @@ class Project < ActiveRecord::Base
     import_type == 'gitea'
   end
 
-  def github_import?
-    import_type == 'github'
-  end
-
   def check_limit
     unless creator.can_create_project? || namespace.kind == 'group'
       projects_limit = creator.projects_limit
@@ -1044,6 +1043,18 @@ class Project < ActiveRecord::Base
     forked_from_project || fork_network&.root_project
   end
 
+  def lfs_storage_project
+    @lfs_storage_project ||= begin
+      result = self
+
+      # TODO: Make this go to the fork_network root immeadiatly
+      # dependant on the discussion in: https://gitlab.com/gitlab-org/gitlab-ce/issues/39769
+      result = result.fork_source while result&.forked?
+
+      result || self
+    end
+  end
+
   def personal?
     !group
   end
@@ -1188,6 +1199,10 @@ class Project < ActiveRecord::Base
     !!repository.exists?
   end
 
+  def wiki_repository_exists?
+    wiki.repository_exists?
+  end
+
   # update visibility_level of forks
   def update_forks_visibility_level
     return unless visibility_level < visibility_level_was
@@ -1431,6 +1446,31 @@ class Project < ActiveRecord::Base
     reload_repository!
   end
 
+  def after_import
+    repository.after_import
+    import_finish
+    remove_import_jid
+    update_project_counter_caches
+  end
+
+  def update_project_counter_caches
+    classes = [
+      Projects::OpenIssuesCountService,
+      Projects::OpenMergeRequestsCountService
+    ]
+
+    classes.each do |klass|
+      klass.new(self).refresh_cache
+    end
+  end
+
+  def remove_import_jid
+    return unless import_jid
+
+    Gitlab::SidekiqStatus.unset(import_jid)
+    update_column(:import_jid, nil)
+  end
+
   def running_or_pending_build_count(force: false)
     Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do
       builds.running_or_pending.count(:all)
@@ -1688,6 +1728,17 @@ class Project < ActiveRecord::Base
     Gitlab::ReferenceCounter.new(gl_repository(is_wiki: wiki))
   end
 
+  # Refreshes the expiration time of the associated import job ID.
+  #
+  # This method can be used by asynchronous importers to refresh the status,
+  # preventing the StuckImportJobsWorker from marking the import as failed.
+  def refresh_import_jid_expiration
+    return unless import_jid
+
+    Gitlab::SidekiqStatus
+      .set(import_jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION)
+  end
+
   private
 
   def storage
diff --git a/app/models/project_custom_attribute.rb b/app/models/project_custom_attribute.rb
new file mode 100644
index 00000000000..3f1a7b86a82
--- /dev/null
+++ b/app/models/project_custom_attribute.rb
@@ -0,0 +1,6 @@
+class ProjectCustomAttribute < ActiveRecord::Base
+  belongs_to :project
+
+  validates :project, :key, :value, presence: true
+  validates :key, uniqueness: { scope: [:project_id] }
+end
diff --git a/app/models/project_services/chat_message/issue_message.rb b/app/models/project_services/chat_message/issue_message.rb
index 1327b075858..3273f41dbd2 100644
--- a/app/models/project_services/chat_message/issue_message.rb
+++ b/app/models/project_services/chat_message/issue_message.rb
@@ -39,7 +39,7 @@ module ChatMessage
     private
 
     def message
-      if state == 'opened'
+      if opened_issue?
         "[#{project_link}] Issue #{state} by #{user_combined_name}"
       else
         "[#{project_link}] Issue #{issue_link} #{state} by #{user_combined_name}"
diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb
index 5c0b3338a62..5080acffb3c 100644
--- a/app/models/project_services/kubernetes_service.rb
+++ b/app/models/project_services/kubernetes_service.rb
@@ -136,6 +136,10 @@ class KubernetesService < DeploymentService
     { pods: read_pods }
   end
 
+  def kubeclient
+    @kubeclient ||= build_kubeclient!
+  end
+
   TEMPLATE_PLACEHOLDER = 'Kubernetes namespace'.freeze
 
   private
diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb
index 217f753f05f..fa7b3f2bcaf 100644
--- a/app/models/project_services/prometheus_service.rb
+++ b/app/models/project_services/prometheus_service.rb
@@ -25,7 +25,7 @@ class PrometheusService < MonitoringService
   end
 
   def description
-    'Prometheus monitoring'
+    s_('PrometheusService|Prometheus monitoring')
   end
 
   def self.to_param
@@ -38,8 +38,8 @@ class PrometheusService < MonitoringService
         type: 'text',
         name: 'api_url',
         title: 'API URL',
-        placeholder: 'Prometheus API Base URL, like http://prometheus.example.com/',
-        help: 'By default, Prometheus listens on ‘http://localhost:9090’. It’s not recommended to change the default address and port as this might affect or conflict with other services running on the GitLab server.',
+        placeholder: s_('PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/'),
+        help: s_('PrometheusService|By default, Prometheus listens on ‘http://localhost:9090’. It’s not recommended to change the default address and port as this might affect or conflict with other services running on the GitLab server.'),
         required: true
       }
     ]
diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb
index 43de6809178..3eecbea8cbf 100644
--- a/app/models/project_wiki.rb
+++ b/app/models/project_wiki.rb
@@ -21,7 +21,7 @@ class ProjectWiki
   end
 
   delegate :empty?, to: :pages
-  delegate :repository_storage_path, to: :project
+  delegate :repository_storage_path, :hashed_storage?, to: :project
 
   def path
     @project.path + '.wiki'
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 69cddb36b2e..3a89fa9264b 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -906,13 +906,13 @@ class Repository
     branch = Gitlab::Git::Branch.find(self, branch_or_name)
 
     if branch
-      root_ref_sha = commit(root_ref).sha
-      same_head = branch.target == root_ref_sha
+      @root_ref_sha ||= commit(root_ref).sha
+      same_head = branch.target == @root_ref_sha
       merged =
         if pre_loaded_merged_branches
           pre_loaded_merged_branches.include?(branch.name)
         else
-          ancestor?(branch.target, root_ref_sha)
+          ancestor?(branch.target, @root_ref_sha)
         end
 
       !same_head && merged
@@ -969,8 +969,12 @@ class Repository
     gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
   end
 
-  def fetch_source_branch(source_repository, source_branch, local_ref)
-    raw_repository.fetch_source_branch(source_repository.raw_repository, source_branch, local_ref)
+  def fetch_source_branch!(source_repository, source_branch, local_ref)
+    raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref)
+  end
+
+  def remote_exists?(name)
+    raw_repository.remote_exists?(name)
   end
 
   def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:)
@@ -1058,6 +1062,10 @@ class Repository
     blob_data_at(sha, path)
   end
 
+  def fetch_ref(source_repository, source_ref:, target_ref:)
+    raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref)
+  end
+
   private
 
   # TODO Generice finder, later split this on finders by Ref or Oid
diff --git a/app/models/user.rb b/app/models/user.rb
index bcda4564595..be8112749bf 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -146,7 +146,7 @@ class User < ActiveRecord::Base
     presence: true,
     numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: Gitlab::Database::MAX_INT_VALUE }
   validates :username,
-    dynamic_path: true,
+    user_path: true,
     presence: true,
     uniqueness: { case_sensitive: false }
 
@@ -164,12 +164,13 @@ class User < ActiveRecord::Base
   before_validation :set_notification_email, if: :email_changed?
   before_validation :set_public_email, if: :public_email_changed?
   before_save :ensure_incoming_email_token
-  before_save :ensure_user_rights_and_limits, if: :external_changed?
+  before_save :ensure_user_rights_and_limits, if: ->(user) { user.new_record? || user.external_changed? }
   before_save :skip_reconfirmation!, if: ->(user) { user.email_changed? && user.read_only_attribute?(:email) }
   before_save :check_for_verified_email, if: ->(user) { user.email_changed? && !user.new_record? }
   after_save :ensure_namespace_correct
   after_update :username_changed_hook, if: :username_changed?
   after_destroy :post_destroy_hook
+  after_destroy :remove_key_cache
   after_commit :update_emails_with_primary_email, on: :update, if: -> { previous_changes.key?('email') }
   after_commit :update_invalid_gpg_signatures, on: :update, if: -> { previous_changes.key?('email') }
 
@@ -267,18 +268,23 @@ class User < ActiveRecord::Base
       end
     end
 
+    def for_github_id(id)
+      joins(:identities)
+        .where(identities: { provider: :github, extern_uid: id.to_s })
+    end
+
     # Find a User by their primary email or any associated secondary email
     def find_by_any_email(email)
-      sql = 'SELECT *
-      FROM users
-      WHERE id IN (
-        SELECT id FROM users WHERE email = :email
-        UNION
-        SELECT emails.user_id FROM emails WHERE email = :email
-      )
-      LIMIT 1;'
+      by_any_email(email).take
+    end
+
+    # Returns a relation containing all the users for the given Email address
+    def by_any_email(email)
+      users = where(email: email)
+      emails = joins(:emails).where(emails: { email: email })
+      union = Gitlab::SQL::Union.new([users, emails])
 
-      User.find_by_sql([sql, { email: email }]).first
+      from("(#{union.to_sql}) #{table_name}")
     end
 
     def filter(filter_name)
@@ -619,7 +625,9 @@ class User < ActiveRecord::Base
   end
 
   def require_ssh_key?
-    keys.count == 0 && Gitlab::ProtocolAccess.allowed?('ssh')
+    count = Users::KeysCountService.new(self).count
+
+    count.zero? && Gitlab::ProtocolAccess.allowed?('ssh')
   end
 
   def require_password_creation?
@@ -881,6 +889,10 @@ class User < ActiveRecord::Base
     system_hook_service.execute_hooks_for(self, :destroy)
   end
 
+  def remove_key_cache
+    Users::KeysCountService.new(self).delete_cache
+  end
+
   def delete_async(deleted_by:, params: {})
     block if params[:hard_delete]
     DeleteUserWorker.perform_async(deleted_by.id, id, params)
@@ -916,7 +928,16 @@ class User < ActiveRecord::Base
   end
 
   def manageable_namespaces
-    @manageable_namespaces ||= [namespace] + owned_groups + masters_groups
+    @manageable_namespaces ||= [namespace] + manageable_groups
+  end
+
+  def manageable_groups
+    union = Gitlab::SQL::Union.new([owned_groups.select(:id),
+                                    masters_groups.select(:id)])
+    arel_union = Arel::Nodes::SqlLiteral.new(union.to_sql)
+    owned_and_master_groups = Group.where(Group.arel_table[:id].in(arel_union))
+
+    Gitlab::GroupHierarchy.new(owned_and_master_groups).base_and_descendants
   end
 
   def namespaces
@@ -1139,8 +1160,9 @@ class User < ActiveRecord::Base
       self.can_create_group = false
       self.projects_limit   = 0
     else
-      self.can_create_group = gitlab_config.default_can_create_group
-      self.projects_limit = current_application_settings.default_projects_limit
+      # Only revert these back to the default if they weren't specifically changed in this update.
+      self.can_create_group = gitlab_config.default_can_create_group unless can_create_group_changed?
+      self.projects_limit = current_application_settings.default_projects_limit unless projects_limit_changed?
     end
   end