Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (1723 commits)
  Resolve "Editor icons"
  Refactor issuable destroy action
  Ignore routes matching legacy_*_redirect in route specs
  Gitlab::Git::RevList and LfsChanges use lazy popen
  Gitlab::Git::Popen can lazily hand output to a block
  Merge branch 'master-i18n' into 'master'
  Remove unique validation from external_url in Environment
  Expose `duration` in Job API entity
  Add TimeCop freeze for DST and Regular time
  Harcode project visibility
  update a changelog
  Put a condition to old migration that adds fast_forward column to MRs
  Expose project visibility as CI variable
  fix flaky tests by removing unneeded clicks and focus actions
  fix flaky test in gfm_autocomplete_spec.rb
  Use Gitlab::Git operations for repository mirroring
  Encapsulate git operations for mirroring in Gitlab::Git
  Create a Wiki Repository's raw_repository properly
  Add `Gitlab::Git::Repository#fetch` command
  Fix Gitlab::Metrics::System#real_time and #monotonic_time doc
  ...

70 files changed, 1275 insertions, 354 deletions

diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index c0cc60d5ebf..5e16badabec 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -33,6 +33,8 @@ class ApplicationSetting < ActiveRecord::Base
 
   attr_accessor :domain_whitelist_raw, :domain_blacklist_raw
 
+  default_value_for :id, 1
+
   validates :uuid, presence: true
 
   validates :session_expire_delay,
@@ -151,6 +153,25 @@ class ApplicationSetting < ActiveRecord::Base
             presence: true,
             numericality: { greater_than_or_equal_to: 0 }
 
+  validates :circuitbreaker_backoff_threshold,
+            :circuitbreaker_failure_count_threshold,
+            :circuitbreaker_failure_wait_time,
+            :circuitbreaker_failure_reset_time,
+            :circuitbreaker_storage_timeout,
+            presence: true,
+            numericality: { only_integer: true, greater_than_or_equal_to: 0 }
+
+  validates :circuitbreaker_access_retries,
+            presence: true,
+            numericality: { only_integer: true, greater_than_or_equal_to: 1 }
+
+  validates_each :circuitbreaker_backoff_threshold do |record, attr, value|
+    if value.to_i >= record.circuitbreaker_failure_count_threshold
+      record.errors.add(attr, _("The circuitbreaker backoff threshold should be "\
+                                "lower than the failure count threshold"))
+    end
+  end
+
   SUPPORTED_KEY_TYPES.each do |type|
     validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type }
   end
@@ -194,7 +215,10 @@ class ApplicationSetting < ActiveRecord::Base
     ensure_cache_setup
 
     Rails.cache.fetch(CACHE_KEY) do
-      ApplicationSetting.last
+      ApplicationSetting.last.tap do |settings|
+        # do not cache nils
+        raise 'missing settings' unless settings
+      end
     end
   rescue
     # Fall back to an uncached value if there are any problems (e.g. redis down)
@@ -396,7 +420,7 @@ class ApplicationSetting < ActiveRecord::Base
   #   the enabling/disabling is `performance_bar_allowed_group_id`
   # - If `enable` is false, we set `performance_bar_allowed_group_id` to `nil`
   def performance_bar_enabled=(enable)
-    return if enable
+    return if Gitlab::Utils.to_boolean(enable)
 
     self.performance_bar_allowed_group_id = nil
   end
diff --git a/app/models/blob.rb b/app/models/blob.rb
index 954d4e4d779..ad0bc2e2ead 100644
--- a/app/models/blob.rb
+++ b/app/models/blob.rb
@@ -156,7 +156,9 @@ class Blob < SimpleDelegator
   end
 
   def file_type
-    Gitlab::FileDetector.type_of(path)
+    name = File.basename(path)
+
+    Gitlab::FileDetector.type_of(path) || Gitlab::FileDetector.type_of(name)
   end
 
   def video?
diff --git a/app/models/ci/artifact_blob.rb b/app/models/ci/artifact_blob.rb
index b35febc9ac5..ec56cc53aea 100644
--- a/app/models/ci/artifact_blob.rb
+++ b/app/models/ci/artifact_blob.rb
@@ -2,6 +2,8 @@ module Ci
   class ArtifactBlob
     include BlobLike
 
+    EXTENSIONS_SERVED_BY_PAGES = %w[.html .htm .txt .json].freeze
+
     attr_reader :entry
 
     def initialize(entry)
@@ -17,6 +19,7 @@ module Ci
     def size
       entry.metadata[:size]
     end
+    alias_method :external_size, :size
 
     def data
       "Build artifact #{path}"
@@ -30,6 +33,32 @@ module Ci
       :build_artifact
     end
 
-    alias_method :external_size, :size
+    def external_url(project, job)
+      return unless external_link?(job)
+
+      full_path_parts = project.full_path_components
+      top_level_group = full_path_parts.shift
+
+      artifact_path = [
+        '-', *full_path_parts, '-',
+        'jobs', job.id,
+        'artifacts', path
+      ].join('/')
+
+      "#{pages_config.protocol}://#{top_level_group}.#{pages_config.host}/#{artifact_path}"
+    end
+
+    def external_link?(job)
+      pages_config.enabled &&
+        pages_config.artifacts_server &&
+        EXTENSIONS_SERVED_BY_PAGES.include?(File.extname(name)) &&
+        job.project.public?
+    end
+
+    private
+
+    def pages_config
+      Gitlab.config.pages
+    end
   end
 end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index ee544d8ac56..6ca46ae89c1 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -11,6 +11,7 @@ module Ci
 
     has_many :deployments, as: :deployable
     has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment'
+    has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
 
     # The "environment" field for builds is a String, and is the unexpanded name
     def persisted_environment
@@ -229,6 +230,10 @@ module Ci
       variables
     end
 
+    def features
+      { trace_sections: true }
+    end
+
     def merge_request
       return @merge_request if defined?(@merge_request)
 
@@ -261,6 +266,10 @@ module Ci
       update_attributes(coverage: coverage) if coverage.present?
     end
 
+    def parse_trace_sections!
+      ExtractSectionsFromBuildTraceService.new(project, user).execute(self)
+    end
+
     def trace
       Gitlab::Ci::Trace.new(self)
     end
diff --git a/app/models/ci/build_trace_section.rb b/app/models/ci/build_trace_section.rb
new file mode 100644
index 00000000000..ccdb95546c8
--- /dev/null
+++ b/app/models/ci/build_trace_section.rb
@@ -0,0 +1,11 @@
+module Ci
+  class BuildTraceSection < ActiveRecord::Base
+    extend Gitlab::Ci::Model
+
+    belongs_to :build, class_name: 'Ci::Build'
+    belongs_to :project
+    belongs_to :section_name, class_name: 'Ci::BuildTraceSectionName'
+
+    validates :section_name, :build, :project, presence: true, allow_blank: false
+  end
+end
diff --git a/app/models/ci/build_trace_section_name.rb b/app/models/ci/build_trace_section_name.rb
new file mode 100644
index 00000000000..0fdcb1ea329
--- /dev/null
+++ b/app/models/ci/build_trace_section_name.rb
@@ -0,0 +1,11 @@
+module Ci
+  class BuildTraceSectionName < ActiveRecord::Base
+    extend Gitlab::Ci::Model
+
+    belongs_to :project
+    has_many :trace_sections, class_name: 'Ci::BuildTraceSection', foreign_key: :section_name_id
+
+    validates :name, :project, presence: true, allow_blank: false
+    validates :name, uniqueness: { scope: :project_id }
+  end
+end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index acaa028eaa2..ca65e81f27a 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -5,6 +5,7 @@ module Ci
     include Importable
     include AfterCommitQueue
     include Presentable
+    include Gitlab::OptimisticLocking
 
     belongs_to :project
     belongs_to :user
@@ -58,6 +59,11 @@ module Ci
       auto_devops_source: 2
     }
 
+    enum failure_reason: {
+      unknown_failure: 0,
+      config_error: 1
+    }
+
     state_machine :status, initial: :created do
       event :enqueue do
         transition created: :pending
@@ -109,6 +115,12 @@ module Ci
         pipeline.auto_canceled_by = nil
       end
 
+      before_transition any => :failed do |pipeline, transition|
+        transition.args.first.try do |reason|
+          pipeline.failure_reason = reason
+        end
+      end
+
       after_transition [:created, :pending] => :running do |pipeline|
         pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
       end
@@ -237,9 +249,7 @@ module Ci
     end
 
     def commit
-      @commit ||= project.commit(sha)
-    rescue
-      nil
+      @commit ||= project.commit_by(oid: sha)
     end
 
     def branch?
@@ -263,7 +273,7 @@ module Ci
     end
 
     def cancel_running
-      Gitlab::OptimisticLocking.retry_lock(cancelable_statuses) do |cancelable|
+      retry_optimistic_lock(cancelable_statuses) do |cancelable|
         cancelable.find_each do |job|
           yield(job) if block_given?
           job.cancel
@@ -312,6 +322,10 @@ module Ci
       @stage_seeds ||= config_processor.stage_seeds(self)
     end
 
+    def seeds_size
+      @seeds_size ||= stage_seeds.sum(&:size)
+    end
+
     def has_kubernetes_active?
       project.kubernetes_service&.active?
     end
@@ -403,7 +417,7 @@ module Ci
     end
 
     def update_status
-      Gitlab::OptimisticLocking.retry_lock(self) do
+      retry_optimistic_lock(self) do
         case latest_builds_status
         when 'pending' then enqueue
         when 'running' then run
@@ -434,7 +448,7 @@ module Ci
     def update_duration
       return unless started_at
 
-      self.duration = Gitlab::Ci::PipelineDuration.from_pipeline(self)
+      self.duration = Gitlab::Ci::Pipeline::Duration.from_pipeline(self)
     end
 
     def execute_hooks
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index a0d07902ba2..c6509f89117 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -174,7 +174,7 @@ module Ci
     end
 
     def assignable_for?(project)
-      !locked? || projects.exists?(id: project.id)
+      is_shared? || projects.exists?(id: project.id)
     end
 
     def accepting_tags?(build)
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 2ae8890c1b3..6dba154a6ea 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -25,8 +25,8 @@ class Commit
   DIFF_HARD_LIMIT_FILES = 1000
   DIFF_HARD_LIMIT_LINES = 50000
 
-  # The SHA can be between 7 and 40 hex characters.
-  COMMIT_SHA_PATTERN = '\h{7,40}'.freeze
+  MIN_SHA_LENGTH = 7
+  COMMIT_SHA_PATTERN = /\h{#{MIN_SHA_LENGTH},40}/.freeze
 
   def banzai_render_context(field)
     context = { pipeline: :single_line, project: self.project }
@@ -53,7 +53,7 @@ class Commit
 
     # Truncate sha to 8 characters
     def truncate_sha(sha)
-      sha[0..7]
+      sha[0..MIN_SHA_LENGTH]
     end
 
     def max_diff_options
@@ -100,7 +100,7 @@ class Commit
   def self.reference_pattern
     @reference_pattern ||= %r{
       (?:#{Project.reference_pattern}#{reference_prefix})?
-      (?<commit>\h{7,40})
+      (?<commit>#{COMMIT_SHA_PATTERN})
     }x
   end
 
@@ -216,9 +216,8 @@ class Commit
     @raw.respond_to?(method, include_private) || super
   end
 
-  # Truncate sha to 8 characters
   def short_id
-    @raw.short_id(7)
+    @raw.short_id(MIN_SHA_LENGTH)
   end
 
   def diff_refs
diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb
index 8fbfed11bdf..2ec70203710 100644
--- a/app/models/concerns/avatarable.rb
+++ b/app/models/concerns/avatarable.rb
@@ -11,7 +11,7 @@ module Avatarable
 
     # If asset_host is set then it is expected that assets are handled by a standalone host.
     # That means we do not want to get GitLab's relative_url_root option anymore.
-    host = asset_host.present? ? asset_host : gitlab_host
+    host = (asset_host.present? && (!respond_to?(:public?) || public?)) ? asset_host : gitlab_host
 
     [host, avatar.url].join
   end
diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb
index 193e459977a..98776eab424 100644
--- a/app/models/concerns/cache_markdown_field.rb
+++ b/app/models/concerns/cache_markdown_field.rb
@@ -49,7 +49,8 @@ module CacheMarkdownField
 
     # Always include a project key, or Banzai complains
     project = self.project if self.respond_to?(:project)
-    context = cached_markdown_fields[field].merge(project: project)
+    group = self.group if self.respond_to?(:group)
+    context = cached_markdown_fields[field].merge(project: project, group: group)
 
     # Banzai is less strict about authors, so don't always have an author key
     context[:author] = self.author if self.respond_to?(:author)
@@ -59,7 +60,7 @@ module CacheMarkdownField
 
   # Update every column in a row if any one is invalidated, as we only store
   # one version per row
-  def refresh_markdown_cache!(do_update: false)
+  def refresh_markdown_cache
     options = { skip_project_check: skip_project_check? }
 
     updates = cached_markdown_fields.markdown_fields.map do |markdown_field|
@@ -71,8 +72,14 @@ module CacheMarkdownField
     updates['cached_markdown_version'] = CacheMarkdownField::CACHE_VERSION
 
     updates.each {|html_field, data| write_attribute(html_field, data) }
+  end
+
+  def refresh_markdown_cache!
+    updates = refresh_markdown_cache
+
+    return unless persisted? && Gitlab::Database.read_write?
 
-    update_columns(updates) if persisted? && do_update
+    update_columns(updates)
   end
 
   def cached_html_up_to_date?(markdown_field)
@@ -124,8 +131,8 @@ module CacheMarkdownField
     end
 
     # Using before_update here conflicts with elasticsearch-model somehow
-    before_create :refresh_markdown_cache!, if: :invalidated_markdown_cache?
-    before_update :refresh_markdown_cache!, if: :invalidated_markdown_cache?
+    before_create :refresh_markdown_cache, if: :invalidated_markdown_cache?
+    before_update :refresh_markdown_cache, if: :invalidated_markdown_cache?
   end
 
   class_methods do
diff --git a/app/models/concerns/discussion_on_diff.rb b/app/models/concerns/discussion_on_diff.rb
index eee1a36ac6b..f5cbb3becad 100644
--- a/app/models/concerns/discussion_on_diff.rb
+++ b/app/models/concerns/discussion_on_diff.rb
@@ -28,6 +28,10 @@ module DiscussionOnDiff
     true
   end
 
+  def file_new_path
+    first_note.position.new_path
+  end
+
   # Returns an array of at most 16 highlighted lines above a diff note
   def truncated_diff_lines(highlight: true)
     lines = highlight ? highlighted_diff_lines : diff_lines
diff --git a/app/models/concerns/group_descendant.rb b/app/models/concerns/group_descendant.rb
new file mode 100644
index 00000000000..01957da0bf3
--- /dev/null
+++ b/app/models/concerns/group_descendant.rb
@@ -0,0 +1,56 @@
+module GroupDescendant
+  # Returns the hierarchy of a project or group in the from of a hash upto a
+  # given top.
+  #
+  # > project.hierarchy
+  # => { parent_group => { child_group => project } }
+  def hierarchy(hierarchy_top = nil, preloaded = nil)
+    preloaded ||= ancestors_upto(hierarchy_top)
+    expand_hierarchy_for_child(self, self, hierarchy_top, preloaded)
+  end
+
+  # Merges all hierarchies of the given groups or projects into an array of
+  # hashes. All ancestors need to be loaded into the given `descendants` to avoid
+  # queries down the line.
+  #
+  # > GroupDescendant.merge_hierarchy([project, child_group, child_group2, parent])
+  # => { parent => [{ child_group => project}, child_group2] }
+  def self.build_hierarchy(descendants, hierarchy_top = nil)
+    descendants = Array.wrap(descendants).uniq
+    return [] if descendants.empty?
+
+    unless descendants.all? { |hierarchy| hierarchy.is_a?(GroupDescendant) }
+      raise ArgumentError.new('element is not a hierarchy')
+    end
+
+    all_hierarchies = descendants.map do |descendant|
+      descendant.hierarchy(hierarchy_top, descendants)
+    end
+
+    Gitlab::Utils::MergeHash.merge(all_hierarchies)
+  end
+
+  private
+
+  def expand_hierarchy_for_child(child, hierarchy, hierarchy_top, preloaded)
+    parent = hierarchy_top if hierarchy_top && child.parent_id == hierarchy_top.id
+    parent ||= preloaded.detect { |possible_parent| possible_parent.is_a?(Group) && possible_parent.id == child.parent_id }
+
+    if parent.nil? && !child.parent_id.nil?
+      raise ArgumentError.new('parent was not preloaded')
+    end
+
+    if parent.nil? && hierarchy_top.present?
+      raise ArgumentError.new('specified top is not part of the tree')
+    end
+
+    if parent && parent != hierarchy_top
+      expand_hierarchy_for_child(parent,
+                                 { parent => hierarchy },
+                                 hierarchy_top,
+                                 preloaded)
+    else
+      hierarchy
+    end
+  end
+end
diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb
index 3803e18a96e..7c3ed96bc28 100644
--- a/app/models/concerns/has_status.rb
+++ b/app/models/concerns/has_status.rb
@@ -81,6 +81,7 @@ module HasStatus
     scope :canceled, -> { where(status: 'canceled')  }
     scope :skipped, -> { where(status: 'skipped')  }
     scope :manual, -> { where(status: 'manual')  }
+    scope :alive, -> { where(status: [:created, :pending, :running]) }
     scope :created_or_pending, -> { where(status: [:created, :pending]) }
     scope :running_or_pending, -> { where(status: [:running, :pending]) }
     scope :finished, -> { where(status: [:success, :failed, :canceled]) }
diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb
index 265f6e48540..a928b9d6367 100644
--- a/app/models/concerns/issuable.rb
+++ b/app/models/concerns/issuable.rb
@@ -14,7 +14,6 @@ module Issuable
   include StripAttribute
   include Awardable
   include Taskable
-  include TimeTrackable
   include Importable
   include Editable
   include AfterCommitQueue
@@ -95,8 +94,6 @@ module Issuable
 
     strip_attributes :title
 
-    acts_as_paranoid
-
     after_save :record_metrics, unless: :imported?
 
     # We want to use optimistic lock for cases when only title or description are involved
@@ -143,16 +140,18 @@ module Issuable
     end
 
     def sort(method, excluded_labels: [])
-      sorted = case method.to_s
-               when 'milestone_due_asc' then order_milestone_due_asc
-               when 'milestone_due_desc' then order_milestone_due_desc
-               when 'downvotes_desc' then order_downvotes_desc
-               when 'upvotes_desc' then order_upvotes_desc
-               when 'label_priority' then order_labels_priority(excluded_labels: excluded_labels)
-               when 'priority' then order_due_date_and_labels_priority(excluded_labels: excluded_labels)
-               else
-                 order_by(method)
-               end
+      sorted =
+        case method.to_s
+        when 'downvotes_desc'     then order_downvotes_desc
+        when 'label_priority'     then order_labels_priority(excluded_labels: excluded_labels)
+        when 'milestone'          then order_milestone_due_asc
+        when 'milestone_due_asc'  then order_milestone_due_asc
+        when 'milestone_due_desc' then order_milestone_due_desc
+        when 'popularity'         then order_upvotes_desc
+        when 'priority'           then order_due_date_and_labels_priority(excluded_labels: excluded_labels)
+        when 'upvotes_desc'       then order_upvotes_desc
+        else order_by(method)
+        end
 
       # Break ties with the ID column for pagination
       sorted.order(id: :desc)
@@ -214,7 +213,7 @@ module Issuable
     def grouping_columns(sort)
       grouping_columns = [arel_table[:id]]
 
-      if %w(milestone_due_desc milestone_due_asc).include?(sort)
+      if %w(milestone_due_desc milestone_due_asc milestone).include?(sort)
         milestone_table = Milestone.arel_table
         grouping_columns << milestone_table[:id]
         grouping_columns << milestone_table[:due_date]
@@ -254,23 +253,22 @@ module Issuable
     participants(user).include?(user)
   end
 
-  def to_hook_data(user)
-    hook_data = {
-      object_kind: self.class.name.underscore,
-      user: user.hook_attrs,
-      project: project.hook_attrs,
-      object_attributes: hook_attrs,
-      labels: labels.map(&:hook_attrs),
-      # DEPRECATED
-      repository: project.hook_attrs.slice(:name, :url, :description, :homepage)
-    }
-    if self.is_a?(Issue)
-      hook_data[:assignees] = assignees.map(&:hook_attrs) if assignees.any?
-    else
-      hook_data[:assignee] = assignee.hook_attrs if assignee
+  def to_hook_data(user, old_labels: [], old_assignees: [])
+    changes = previous_changes
+
+    if old_labels != labels
+      changes[:labels] = [old_labels.map(&:hook_attrs), labels.map(&:hook_attrs)]
+    end
+
+    if old_assignees != assignees
+      if self.is_a?(Issue)
+        changes[:assignees] = [old_assignees.map(&:hook_attrs), assignees.map(&:hook_attrs)]
+      else
+        changes[:assignee] = [old_assignees&.first&.hook_attrs, assignee&.hook_attrs]
+      end
     end
 
-    hook_data
+    Gitlab::HookData::IssuableBuilder.new(self).build(user: user, changes: changes)
   end
 
   def labels_array
diff --git a/app/models/concerns/loaded_in_group_list.rb b/app/models/concerns/loaded_in_group_list.rb
new file mode 100644
index 00000000000..dcb3b2b5ff3
--- /dev/null
+++ b/app/models/concerns/loaded_in_group_list.rb
@@ -0,0 +1,72 @@
+module LoadedInGroupList
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def with_counts(archived:)
+      selects_including_counts = [
+        'namespaces.*',
+        "(#{project_count_sql(archived).to_sql}) AS preloaded_project_count",
+        "(#{member_count_sql.to_sql}) AS preloaded_member_count",
+        "(#{subgroup_count_sql.to_sql}) AS preloaded_subgroup_count"
+      ]
+
+      select(selects_including_counts)
+    end
+
+    def with_selects_for_list(archived: nil)
+      with_route.with_counts(archived: archived)
+    end
+
+    private
+
+    def project_count_sql(archived = nil)
+      projects = Project.arel_table
+      namespaces = Namespace.arel_table
+
+      base_count = projects.project(Arel.star.count.as('preloaded_project_count'))
+                     .where(projects[:namespace_id].eq(namespaces[:id]))
+      if archived == 'only'
+        base_count.where(projects[:archived].eq(true))
+      elsif Gitlab::Utils.to_boolean(archived)
+        base_count
+      else
+        base_count.where(projects[:archived].not_eq(true))
+      end
+    end
+
+    def subgroup_count_sql
+      namespaces = Namespace.arel_table
+      children = namespaces.alias('children')
+
+      namespaces.project(Arel.star.count.as('preloaded_subgroup_count'))
+        .from(children)
+        .where(children[:parent_id].eq(namespaces[:id]))
+    end
+
+    def member_count_sql
+      members = Member.arel_table
+      namespaces = Namespace.arel_table
+
+      members.project(Arel.star.count.as('preloaded_member_count'))
+        .where(members[:source_type].eq(Namespace.name))
+        .where(members[:source_id].eq(namespaces[:id]))
+        .where(members[:requested_at].eq(nil))
+    end
+  end
+
+  def children_count
+    @children_count ||= project_count + subgroup_count
+  end
+
+  def project_count
+    @project_count ||= try(:preloaded_project_count) || projects.non_archived.count
+  end
+
+  def subgroup_count
+    @subgroup_count ||= try(:preloaded_subgroup_count) || children.count
+  end
+
+  def member_count
+    @member_count ||= try(:preloaded_member_count) || users.count
+  end
+end
diff --git a/app/models/concerns/noteable.rb b/app/models/concerns/noteable.rb
index 9d81a19cbb9..b44274f6145 100644
--- a/app/models/concerns/noteable.rb
+++ b/app/models/concerns/noteable.rb
@@ -75,4 +75,8 @@ module Noteable
   def discussions_can_be_resolved_by?(user)
     discussions_to_be_resolved.all? { |discussion| discussion.can_resolve?(user) }
   end
+
+  def lockable?
+    [MergeRequest, Issue].include?(self.class)
+  end
 end
diff --git a/app/models/concerns/repository_mirroring.rb b/app/models/concerns/repository_mirroring.rb
deleted file mode 100644
index fed336c29d6..00000000000
--- a/app/models/concerns/repository_mirroring.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-module RepositoryMirroring
-  def set_remote_as_mirror(name)
-    config = raw_repository.rugged.config
-
-    # This is used to define repository as equivalent as "git clone --mirror"
-    config["remote.#{name}.fetch"] = 'refs/*:refs/*'
-    config["remote.#{name}.mirror"] = true
-    config["remote.#{name}.prune"] = true
-  end
-
-  def fetch_mirror(remote, url)
-    add_remote(remote, url)
-    set_remote_as_mirror(remote)
-    fetch_remote(remote, forced: true)
-    remove_remote(remote)
-  end
-end
diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb
index 80a8f63514f..05ddae42d2d 100644
--- a/app/models/concerns/routable.rb
+++ b/app/models/concerns/routable.rb
@@ -107,7 +107,10 @@ module Routable
     RequestStore[full_path_key] ||= uncached_full_path
   end
 
-  # rubocop:disable Cop/ModuleWithInstanceVariables
+  def full_path_components
+    full_path.split('/')
+  end
+
   def expires_full_path_cache
     RequestStore.delete(full_path_key) if RequestStore.active?
     @full_path = nil
@@ -155,6 +158,8 @@ module Routable
   end
 
   def update_route
+    return if Gitlab::Database.read_only?
+
     prepare_route
     route.save
   end
diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb
index db3cd257584..cefa5c13c5f 100644
--- a/app/models/concerns/sortable.rb
+++ b/app/models/concerns/sortable.rb
@@ -19,14 +19,15 @@ module Sortable
   module ClassMethods
     def order_by(method)
       case method.to_s
-      when 'name_asc' then order_name_asc
-      when 'name_desc' then order_name_desc
-      when 'updated_asc' then order_updated_asc
-      when 'updated_desc' then order_updated_desc
-      when 'created_asc' then order_created_asc
+      when 'created_asc'  then order_created_asc
+      when 'created_date' then order_created_desc
       when 'created_desc' then order_created_desc
-      when 'id_desc' then order_id_desc
-      when 'id_asc' then order_id_asc
+      when 'id_asc'       then order_id_asc
+      when 'id_desc'      then order_id_desc
+      when 'name_asc'     then order_name_asc
+      when 'name_desc'    then order_name_desc
+      when 'updated_asc'  then order_updated_asc
+      when 'updated_desc' then order_updated_desc
       else
         all
       end
diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb
index 5ab5c80a2f5..b3020484738 100644
--- a/app/models/concerns/storage/legacy_namespace.rb
+++ b/app/models/concerns/storage/legacy_namespace.rb
@@ -7,6 +7,8 @@ module Storage
         raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry')
       end
 
+      expires_full_path_cache
+
       # Move the namespace directory in all storage paths used by member projects
       repository_storage_paths.each do |repository_storage_path|
         # Ensure old directory exists before moving it
diff --git a/app/models/concerns/subscribable.rb b/app/models/concerns/subscribable.rb
index 274b38a7708..f478c8ede18 100644
--- a/app/models/concerns/subscribable.rb
+++ b/app/models/concerns/subscribable.rb
@@ -13,6 +13,8 @@ module Subscribable
   end
 
   def subscribed?(user, project = nil)
+    return false unless user
+
     if subscription = subscriptions.find_by(user: user, project: project)
       subscription.subscribed
     else
diff --git a/app/models/concerns/time_trackable.rb b/app/models/concerns/time_trackable.rb
index 995fa98efac..49438908c36 100644
--- a/app/models/concerns/time_trackable.rb
+++ b/app/models/concerns/time_trackable.rb
@@ -8,7 +8,7 @@ module TimeTrackable
   extend ActiveSupport::Concern
 
   included do
-    attr_reader :time_spent, :time_spent_user
+    attr_reader :time_spent, :time_spent_user, :spent_at
 
     alias_method :time_spent?, :time_spent
 
@@ -24,6 +24,7 @@ module TimeTrackable
   def spend_time(options)
     @time_spent = options[:duration]
     @time_spent_user = options[:user]
+    @spent_at = options[:spent_at]
     @original_total_time_spent = nil
 
     return if @time_spent == 0
@@ -56,7 +57,11 @@ module TimeTrackable
   end
 
   def add_or_subtract_spent_time
-    timelogs.new(time_spent: time_spent, user: @time_spent_user)
+    timelogs.new(
+      time_spent: time_spent,
+      user: @time_spent_user,
+      spent_at: @spent_at
+    )
   end
 
   # rubocop:disable Cop/ModuleWithInstanceVariables
diff --git a/app/models/concerns/token_authenticatable.rb b/app/models/concerns/token_authenticatable.rb
index a7d5de48c66..ec3543f7053 100644
--- a/app/models/concerns/token_authenticatable.rb
+++ b/app/models/concerns/token_authenticatable.rb
@@ -43,15 +43,17 @@ module TokenAuthenticatable
         write_attribute(token_field, token) if token
       end
 
+      # Returns a token, but only saves when the database is in read & write mode
       define_method("ensure_#{token_field}!") do
         send("reset_#{token_field}!") if read_attribute(token_field).blank? # rubocop:disable GitlabSecurity/PublicSend
 
         read_attribute(token_field)
       end
 
+      # Resets the token, but only saves when the database is in read & write mode
       define_method("reset_#{token_field}!") do
         write_new_token(token_field)
-        save!
+        save! if Gitlab::Database.read_write?
       end
     end
   end
diff --git a/app/models/diff_discussion.rb b/app/models/diff_discussion.rb
index 07c4846e2ac..6eba87da1a1 100644
--- a/app/models/diff_discussion.rb
+++ b/app/models/diff_discussion.rb
@@ -11,6 +11,8 @@ class DiffDiscussion < Discussion
   delegate  :position,
             :original_position,
             :change_position,
+            :on_text?,
+            :on_image?,
 
             to: :first_note
 
diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb
index e9a60e6ce09..d88a92dc027 100644
--- a/app/models/diff_note.rb
+++ b/app/models/diff_note.rb
@@ -12,8 +12,8 @@ class DiffNote < Note
 
   validates :original_position, presence: true
   validates :position, presence: true
-  validates :diff_line, presence: true
-  validates :line_code, presence: true, line_code: true
+  validates :diff_line, presence: true, if: :on_text?
+  validates :line_code, presence: true, line_code: true, if: :on_text?
   validates :noteable_type, inclusion: { in: NOTEABLE_TYPES }
   validate :positions_complete
   validate :verify_supported
@@ -43,6 +43,14 @@ class DiffNote < Note
     end
   end
 
+  def on_text?
+    position.position_type == "text"
+  end
+
+  def on_image?
+    position.position_type == "image"
+  end
+
   def diff_file
     @diff_file ||= self.original_position.diff_file(self.project.repository)
   end
@@ -56,6 +64,8 @@ class DiffNote < Note
   end
 
   def original_line_code
+    return unless on_text?
+
     self.diff_file.line_code(self.diff_line)
   end
 
diff --git a/app/models/discussion.rb b/app/models/discussion.rb
index b80da7b246a..437df923d2d 100644
--- a/app/models/discussion.rb
+++ b/app/models/discussion.rb
@@ -66,6 +66,10 @@ class Discussion
     @context_noteable = context_noteable
   end
 
+  def on_image?
+    false
+  end
+
   def ==(other)
     other.class == self.class &&
       other.context_noteable == self.context_noteable &&
diff --git a/app/models/email.rb b/app/models/email.rb
index 826d4f16edb..2da8b050149 100644
--- a/app/models/email.rb
+++ b/app/models/email.rb
@@ -7,6 +7,15 @@ class Email < ActiveRecord::Base
   validates :email, presence: true, uniqueness: true, email: true
   validate :unique_email, if: ->(email) { email.email_changed? }
 
+  scope :confirmed, -> { where.not(confirmed_at: nil) }
+
+  after_commit :update_invalid_gpg_signatures, if: -> { previous_changes.key?('confirmed_at') }
+
+  devise :confirmable
+  self.reconfirmable = false  # currently email can't be changed, no need to reconfirm
+
+  delegate :username, to: :user
+
   def email=(value)
     write_attribute(:email, value.downcase.strip)
   end
@@ -14,4 +23,9 @@ class Email < ActiveRecord::Base
   def unique_email
     self.errors.add(:email, 'has already been taken') if User.exists?(email: self.email)
   end
+
+  # once email is confirmed, update the gpg signatures
+  def update_invalid_gpg_signatures
+    user.update_invalid_gpg_signatures if confirmed?
+  end
 end
diff --git a/app/models/environment.rb b/app/models/environment.rb
index b6868ccbe8f..21a028e351c 100644
--- a/app/models/environment.rb
+++ b/app/models/environment.rb
@@ -30,7 +30,6 @@ class Environment < ActiveRecord::Base
                       message: Gitlab::Regex.environment_slug_regex_message }
 
   validates :external_url,
-            uniqueness: { scope: :project_id },
             length: { maximum: 255 },
             allow_nil: true,
             addressable_url: true
@@ -110,7 +109,7 @@ class Environment < ActiveRecord::Base
   end
 
   def ref_path
-    "refs/#{Repository::REF_ENVIRONMENTS}/#{Shellwords.shellescape(name)}"
+    "refs/#{Repository::REF_ENVIRONMENTS}/#{slug}"
   end
 
   def formatted_external_url
@@ -164,6 +163,10 @@ class Environment < ActiveRecord::Base
     end
   end
 
+  def slug
+    super.presence || generate_slug
+  end
+
   # An environment name is not necessarily suitable for use in URLs, DNS
   # or other third-party contexts, so provide a slugified version. A slug has
   # the following properties:
diff --git a/app/models/epic.rb b/app/models/epic.rb
new file mode 100644
index 00000000000..62898a02e2d
--- /dev/null
+++ b/app/models/epic.rb
@@ -0,0 +1,7 @@
+# Placeholder class for model that is implemented in EE
+# It will reserve (ee#3853) '&' as a reference prefix, but the table does not exists in CE
+class Epic < ActiveRecord::Base
+  # TODO: this will be implemented as part of #3853
+  def to_reference
+  end
+end
diff --git a/app/models/fork_network.rb b/app/models/fork_network.rb
new file mode 100644
index 00000000000..7f1728e8c77
--- /dev/null
+++ b/app/models/fork_network.rb
@@ -0,0 +1,19 @@
+class ForkNetwork < ActiveRecord::Base
+  belongs_to :root_project, class_name: 'Project'
+  has_many :fork_network_members
+  has_many :projects, through: :fork_network_members
+
+  after_create :add_root_as_member, if: :root_project
+
+  def add_root_as_member
+    projects << root_project
+  end
+
+  def find_forks_in(other_projects)
+    projects.where(id: other_projects)
+  end
+
+  def merge_requests
+    MergeRequest.where(target_project: projects)
+  end
+end
diff --git a/app/models/fork_network_member.rb b/app/models/fork_network_member.rb
new file mode 100644
index 00000000000..6a9b52a1ef8
--- /dev/null
+++ b/app/models/fork_network_member.rb
@@ -0,0 +1,7 @@
+class ForkNetworkMember < ActiveRecord::Base
+  belongs_to :fork_network
+  belongs_to :project
+  belongs_to :forked_from_project, class_name: 'Project'
+
+  validates :fork_network, :project, presence: true
+end
diff --git a/app/models/gcp/cluster.rb b/app/models/gcp/cluster.rb
new file mode 100644
index 00000000000..162a690c0e3
--- /dev/null
+++ b/app/models/gcp/cluster.rb
@@ -0,0 +1,116 @@
+module Gcp
+  class Cluster < ActiveRecord::Base
+    extend Gitlab::Gcp::Model
+    include Presentable
+
+    belongs_to :project, inverse_of: :cluster
+    belongs_to :user
+    belongs_to :service
+
+    scope :enabled, -> { where(enabled: true) }
+    scope :disabled, -> { where(enabled: false) }
+
+    default_value_for :gcp_cluster_zone, 'us-central1-a'
+    default_value_for :gcp_cluster_size, 3
+    default_value_for :gcp_machine_type, 'n1-standard-4'
+
+    attr_encrypted :password,
+      mode: :per_attribute_iv,
+      key: Gitlab::Application.secrets.db_key_base,
+      algorithm: 'aes-256-cbc'
+
+    attr_encrypted :kubernetes_token,
+      mode: :per_attribute_iv,
+      key: Gitlab::Application.secrets.db_key_base,
+      algorithm: 'aes-256-cbc'
+
+    attr_encrypted :gcp_token,
+      mode: :per_attribute_iv,
+      key: Gitlab::Application.secrets.db_key_base,
+      algorithm: 'aes-256-cbc'
+
+    validates :gcp_project_id,
+      length: 1..63,
+      format: {
+        with: Gitlab::Regex.kubernetes_namespace_regex,
+        message: Gitlab::Regex.kubernetes_namespace_regex_message
+      }
+
+    validates :gcp_cluster_name,
+      length: 1..63,
+      format: {
+        with: Gitlab::Regex.kubernetes_namespace_regex,
+        message: Gitlab::Regex.kubernetes_namespace_regex_message
+      }
+
+    validates :gcp_cluster_zone, presence: true
+
+    validates :gcp_cluster_size,
+      presence: true,
+      numericality: {
+        only_integer: true,
+        greater_than: 0
+      }
+
+    validates :project_namespace,
+      allow_blank: true,
+      length: 1..63,
+      format: {
+        with: Gitlab::Regex.kubernetes_namespace_regex,
+        message: Gitlab::Regex.kubernetes_namespace_regex_message
+      }
+
+    # if we do not do status transition we prevent change
+    validate :restrict_modification, on: :update, unless: :status_changed?
+
+    state_machine :status, initial: :scheduled do
+      state :scheduled, value: 1
+      state :creating, value: 2
+      state :created, value: 3
+      state :errored, value: 4
+
+      event :make_creating do
+        transition any - [:creating] => :creating
+      end
+
+      event :make_created do
+        transition any - [:created] => :created
+      end
+
+      event :make_errored do
+        transition any - [:errored] => :errored
+      end
+
+      before_transition any => [:errored, :created] do |cluster|
+        cluster.gcp_token = nil
+        cluster.gcp_operation_id = nil
+      end
+
+      before_transition any => [:errored] do |cluster, transition|
+        status_reason = transition.args.first
+        cluster.status_reason = status_reason if status_reason
+      end
+    end
+
+    def project_namespace_placeholder
+      "#{project.path}-#{project.id}"
+    end
+
+    def on_creation?
+      scheduled? || creating?
+    end
+
+    def api_url
+      'https://' + endpoint if endpoint
+    end
+
+    def restrict_modification
+      if on_creation?
+        errors.add(:base, "cannot modify during creation")
+        return false
+      end
+
+      true
+    end
+  end
+end
diff --git a/app/models/gpg_key.rb b/app/models/gpg_key.rb
index 44deae4234b..44eda741679 100644
--- a/app/models/gpg_key.rb
+++ b/app/models/gpg_key.rb
@@ -9,6 +9,9 @@ class GpgKey < ActiveRecord::Base
 
   belongs_to :user
   has_many :gpg_signatures
+  has_many :subkeys, class_name: 'GpgKeySubkey'
+
+  scope :with_subkeys, -> { includes(:subkeys) }
 
   validates :user, presence: true
 
@@ -36,10 +39,12 @@ class GpgKey < ActiveRecord::Base
 
   before_validation :extract_fingerprint, :extract_primary_keyid
   after_commit :update_invalid_gpg_signatures, on: :create
+  after_create :generate_subkeys
 
   def primary_keyid
     super&.upcase
   end
+  alias_method :keyid, :primary_keyid
 
   def fingerprint
     super&.upcase
@@ -49,6 +54,10 @@ class GpgKey < ActiveRecord::Base
     super(value&.strip)
   end
 
+  def keyids
+    [keyid].concat(subkeys.map(&:keyid))
+  end
+
   def user_infos
     @user_infos ||= Gitlab::Gpg.user_infos_from_key(key)
   end
@@ -73,7 +82,7 @@ class GpgKey < ActiveRecord::Base
   end
 
   def verified_and_belongs_to_email?(email)
-    emails_with_verified_status.fetch(email, false)
+    emails_with_verified_status.fetch(email.downcase, false)
   end
 
   def update_invalid_gpg_signatures
@@ -82,10 +91,11 @@ class GpgKey < ActiveRecord::Base
 
   def revoke
     GpgSignature
-      .where(gpg_key: self)
+      .with_key_and_subkeys(self)
       .where.not(verification_status: GpgSignature.verification_statuses[:unknown_key])
       .update_all(
         gpg_key_id: nil,
+        gpg_key_subkey_id: nil,
         verification_status: GpgSignature.verification_statuses[:unknown_key],
         updated_at: Time.zone.now
       )
@@ -106,4 +116,12 @@ class GpgKey < ActiveRecord::Base
     # only allows one key
     self.primary_keyid = Gitlab::Gpg.primary_keyids_from_key(key).first
   end
+
+  def generate_subkeys
+    gpg_subkeys = Gitlab::Gpg.subkeys_from_key(key)
+
+    gpg_subkeys[primary_keyid]&.each do |subkey_data|
+      subkeys.create!(keyid: subkey_data[:keyid], fingerprint: subkey_data[:fingerprint])
+    end
+  end
 end
diff --git a/app/models/gpg_key_subkey.rb b/app/models/gpg_key_subkey.rb
new file mode 100644
index 00000000000..b57922aba30
--- /dev/null
+++ b/app/models/gpg_key_subkey.rb
@@ -0,0 +1,22 @@
+class GpgKeySubkey < ActiveRecord::Base
+  include ShaAttribute
+
+  sha_attribute :keyid
+  sha_attribute :fingerprint
+
+  belongs_to :gpg_key
+
+  validates :gpg_key_id, presence: true
+  validates :fingerprint, :keyid, presence: true, uniqueness: true
+
+  delegate :key, :user, :user_infos, :verified?, :verified_user_infos,
+    :verified_and_belongs_to_email?, to: :gpg_key
+
+  def keyid
+    super&.upcase
+  end
+
+  def fingerprint
+    super&.upcase
+  end
+end
diff --git a/app/models/gpg_signature.rb b/app/models/gpg_signature.rb
index 1f047a32c84..bf88d75246f 100644
--- a/app/models/gpg_signature.rb
+++ b/app/models/gpg_signature.rb
@@ -15,11 +15,42 @@ class GpgSignature < ActiveRecord::Base
 
   belongs_to :project
   belongs_to :gpg_key
+  belongs_to :gpg_key_subkey
 
   validates :commit_sha, presence: true
   validates :project_id, presence: true
   validates :gpg_key_primary_keyid, presence: true
 
+  def self.with_key_and_subkeys(gpg_key)
+    subkey_ids = gpg_key.subkeys.pluck(:id)
+
+    where(
+      arel_table[:gpg_key_id].eq(gpg_key.id).or(
+        arel_table[:gpg_key_subkey_id].in(subkey_ids)
+      )
+    )
+  end
+
+  def gpg_key=(model)
+    case model
+    when GpgKey
+      super
+    when GpgKeySubkey
+      self.gpg_key_subkey = model
+    when NilClass
+      super
+      self.gpg_key_subkey = nil
+    end
+  end
+
+  def gpg_key
+    if gpg_key_id
+      super
+    elsif gpg_key_subkey_id
+      gpg_key_subkey
+    end
+  end
+
   def gpg_key_primary_keyid
     super&.upcase
   end
@@ -29,6 +60,8 @@ class GpgSignature < ActiveRecord::Base
   end
 
   def gpg_commit
+    return unless commit
+
     Gitlab::Gpg::Commit.new(commit)
   end
 end
diff --git a/app/models/group.rb b/app/models/group.rb
index e746e4a12c9..c660de7fcb6 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -6,6 +6,8 @@ class Group < Namespace
   include Avatarable
   include Referable
   include SelectForProjectAuthorization
+  include LoadedInGroupList
+  include GroupDescendant
 
   has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent
   alias_method :members, :group_members
@@ -40,6 +42,7 @@ class Group < Namespace
   after_create :post_create_hook
   after_destroy :post_destroy_hook
   after_save :update_two_factor_requirement
+  after_update :path_changed_hook, if: :path_changed?
 
   class << self
     def supports_nested_groups?
@@ -178,6 +181,12 @@ class Group < Namespace
     add_user(user, :owner, current_user: current_user)
   end
 
+  def member?(user, min_access_level = Gitlab::Access::GUEST)
+    return false unless user
+
+    max_member_access_for_user(user) >= min_access_level
+  end
+
   def has_owner?(user)
     return false unless user
 
@@ -287,6 +296,12 @@ class Group < Namespace
     list_of_ids.reverse.map { |group| variables[group.id] }.compact.flatten
   end
 
+  def full_path_was
+    return path_was unless has_parent?
+
+    "#{parent.full_path}/#{path_was}"
+  end
+
   private
 
   def update_two_factor_requirement
@@ -295,6 +310,10 @@ class Group < Namespace
     users.find_each(&:update_two_factor_requirement)
   end
 
+  def path_changed_hook
+    system_hook_service.execute_hooks_for(self, :rename)
+  end
+
   def visibility_level_allowed_by_parent
     return if visibility_level_allowed_by_parent?
 
diff --git a/app/models/identity.rb b/app/models/identity.rb
index 920a25932b4..ac8094b610e 100644
--- a/app/models/identity.rb
+++ b/app/models/identity.rb
@@ -7,7 +7,10 @@ class Identity < ActiveRecord::Base
   validates :extern_uid, allow_blank: true, uniqueness: { scope: :provider }
   validates :user_id, uniqueness: { scope: :provider }
 
-  scope :with_extern_uid, ->(provider, extern_uid) { where(extern_uid: extern_uid, provider: provider) }
+  scope :with_extern_uid, ->(provider, extern_uid) do
+    extern_uid = Gitlab::LDAP::Person.normalize_dn(extern_uid) if provider.starts_with?('ldap')
+    where(extern_uid: extern_uid, provider: provider)
+  end
 
   def ldap?
     provider.starts_with?('ldap')
diff --git a/app/models/instance_configuration.rb b/app/models/instance_configuration.rb
new file mode 100644
index 00000000000..b30b707e5fe
--- /dev/null
+++ b/app/models/instance_configuration.rb
@@ -0,0 +1,71 @@
+require 'resolv'
+
+class InstanceConfiguration
+  SSH_ALGORITHMS = %w(DSA ECDSA ED25519 RSA).freeze
+  SSH_ALGORITHMS_PATH = '/etc/ssh/'.freeze
+  CACHE_KEY = 'instance_configuration'.freeze
+  EXPIRATION_TIME = 24.hours
+
+  def settings
+    @configuration ||= Rails.cache.fetch(CACHE_KEY, expires_in: EXPIRATION_TIME) do
+      { ssh_algorithms_hashes: ssh_algorithms_hashes,
+        host: host,
+        gitlab_pages: gitlab_pages,
+        gitlab_ci: gitlab_ci }.deep_symbolize_keys
+    end
+  end
+
+  private
+
+  def ssh_algorithms_hashes
+    SSH_ALGORITHMS.map { |algo| ssh_algorithm_hashes(algo) }.compact
+  end
+
+  def host
+    Settings.gitlab.host
+  end
+
+  def gitlab_pages
+    Settings.pages.to_h.merge(ip_address: resolv_dns(Settings.pages.host))
+  end
+
+  def resolv_dns(dns)
+    Resolv.getaddress(dns)
+  rescue Resolv::ResolvError
+  end
+
+  def gitlab_ci
+    Settings.gitlab_ci
+            .to_h
+            .merge(artifacts_max_size: { value: Settings.artifacts.max_size&.megabytes,
+                                         default: 100.megabytes })
+  end
+
+  def ssh_algorithm_file(algorithm)
+    File.join(SSH_ALGORITHMS_PATH, "ssh_host_#{algorithm.downcase}_key.pub")
+  end
+
+  def ssh_algorithm_hashes(algorithm)
+    content = ssh_algorithm_file_content(algorithm)
+    return unless content.present?
+
+    { name: algorithm,
+      md5: ssh_algorithm_md5(content),
+      sha256: ssh_algorithm_sha256(content) }
+  end
+
+  def ssh_algorithm_file_content(algorithm)
+    file = ssh_algorithm_file(algorithm)
+    return unless File.exist?(file)
+
+    File.read(file)
+  end
+
+  def ssh_algorithm_md5(ssh_file_content)
+    OpenSSL::Digest::MD5.hexdigest(ssh_file_content).scan(/../).join(':')
+  end
+
+  def ssh_algorithm_sha256(ssh_file_content)
+    OpenSSL::Digest::SHA256.hexdigest(ssh_file_content)
+  end
+end
diff --git a/app/models/issue.rb b/app/models/issue.rb
index 92a454300af..fc590f9257e 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -10,6 +10,7 @@ class Issue < ActiveRecord::Base
   include FasterCacheKeys
   include RelativePositioning
   include CreatedAtFilterable
+  include TimeTrackable
 
   DueDateStruct = Struct.new(:title, :name).freeze
   NoDueDate     = DueDateStruct.new('No Due Date', '0').freeze
@@ -74,19 +75,7 @@ class Issue < ActiveRecord::Base
     end
   end
 
-  def hook_attrs
-    assignee_ids = self.assignee_ids
-
-    attrs = {
-      total_time_spent: total_time_spent,
-      human_total_time_spent: human_total_time_spent,
-      human_time_estimate: human_time_estimate,
-      assignee_ids: assignee_ids,
-      assignee_id: assignee_ids.first # This key is deprecated
-    }
-
-    attributes.merge!(attrs)
-  end
+  acts_as_paranoid
 
   def self.reference_prefix
     '#'
@@ -116,7 +105,8 @@ class Issue < ActiveRecord::Base
 
   def self.sort(method, excluded_labels: [])
     case method.to_s
-    when 'due_date_asc' then order_due_date_asc
+    when 'due_date'      then order_due_date_asc
+    when 'due_date_asc'  then order_due_date_asc
     when 'due_date_desc' then order_due_date_desc
     else
       super
@@ -130,6 +120,10 @@ class Issue < ActiveRecord::Base
               "id DESC")
   end
 
+  def hook_attrs
+    Gitlab::HookData::IssueBuilder.new(self).build
+  end
+
   # Returns a Hash of attributes to be used for Twitter card metadata
   def card_attributes
     {
diff --git a/app/models/key.rb b/app/models/key.rb
index 0c41e34d969..f119b15c737 100644
--- a/app/models/key.rb
+++ b/app/models/key.rb
@@ -34,6 +34,7 @@ class Key < ActiveRecord::Base
     value&.delete!("\n\r")
     value.strip! unless value.blank?
     write_attribute(:key, value)
+    @public_key = nil
   end
 
   def publishable_key
diff --git a/app/models/legacy_diff_discussion.rb b/app/models/legacy_diff_discussion.rb
index 3c1d34db5fa..80fc6304fd4 100644
--- a/app/models/legacy_diff_discussion.rb
+++ b/app/models/legacy_diff_discussion.rb
@@ -17,6 +17,14 @@ class LegacyDiffDiscussion < Discussion
     true
   end
 
+  def on_image?
+    false
+  end
+
+  def on_text?
+    true
+  end
+
   def active?(*args)
     return @active if @active.present?
 
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 8d9a30397a9..3133dc9e7eb 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -6,6 +6,7 @@ class MergeRequest < ActiveRecord::Base
   include Sortable
   include IgnorableColumn
   include CreatedAtFilterable
+  include TimeTrackable
 
   ignore_column :locked_at
 
@@ -119,6 +120,8 @@ class MergeRequest < ActiveRecord::Base
 
   after_save :keep_around_commit
 
+  acts_as_paranoid
+
   def self.reference_prefix
     '!'
   end
@@ -179,6 +182,10 @@ class MergeRequest < ActiveRecord::Base
     work_in_progress?(title) ? title : "WIP: #{title}"
   end
 
+  def hook_attrs
+    Gitlab::HookData::MergeRequestBuilder.new(self).build
+  end
+
   # Returns a Hash of attributes to be used for Twitter card metadata
   def card_attributes
     {
@@ -392,7 +399,11 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def merge_ongoing?
-    !!merge_jid && !merged?
+    # While the MergeRequest is locked, it should present itself as 'merge ongoing'.
+    # The unlocking process is handled by StuckMergeJobsWorker scheduled in Cron.
+    return true if locked?
+
+    !!merge_jid && !merged? && Gitlab::SidekiqStatus.running?(merge_jid)
   end
 
   def closed_without_fork?
@@ -403,7 +414,7 @@ class MergeRequest < ActiveRecord::Base
     return false unless for_fork?
     return true unless source_project
 
-    !source_project.forked_from?(target_project)
+    !source_project.in_fork_network_of?(target_project)
   end
 
   def reopenable?
@@ -415,6 +426,8 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def create_merge_request_diff
+    fetch_ref
+
     # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37435
     Gitlab::GitalyClient.allow_n_plus_1_calls do
       merge_request_diffs.create
@@ -462,6 +475,7 @@ class MergeRequest < ActiveRecord::Base
     return unless open?
 
     old_diff_refs = self.diff_refs
+
     create_merge_request_diff
     MergeRequests::MergeRequestDiffCacheService.new.execute(self)
     new_diff_refs = self.diff_refs
@@ -474,7 +488,7 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def check_if_can_be_merged
-    return unless unchecked?
+    return unless unchecked? && Gitlab::Database.read_write?
 
     can_be_merged =
       !broken? && project.repository.can_be_merged?(diff_head_sha, target_branch)
@@ -524,6 +538,14 @@ class MergeRequest < ActiveRecord::Base
     true
   end
 
+  def ff_merge_possible?
+    project.repository.ancestor?(target_branch_sha, diff_head_sha)
+  end
+
+  def should_be_rebased?
+    project.ff_merge_must_be_possible? && !ff_merge_possible?
+  end
+
   def can_cancel_merge_when_pipeline_succeeds?(current_user)
     can_be_merged_by?(current_user) || self.author == current_user
   end
@@ -552,14 +574,20 @@ class MergeRequest < ActiveRecord::Base
     commits_for_notes_limit = 100
     commit_ids = commit_shas.take(commits_for_notes_limit)
 
-    Note.where(
-      "(project_id = :target_project_id AND noteable_type = 'MergeRequest' AND noteable_id = :mr_id) OR" +
-      "((project_id = :source_project_id OR project_id = :target_project_id) AND noteable_type = 'Commit' AND commit_id IN (:commit_ids))",
-      mr_id: id,
-      commit_ids: commit_ids,
-      target_project_id: target_project_id,
-      source_project_id: source_project_id
-    )
+    commit_notes = Note
+      .except(:order)
+      .where(project_id: [source_project_id, target_project_id])
+      .where(noteable_type: 'Commit', commit_id: commit_ids)
+
+    # We're using a UNION ALL here since this results in better performance
+    # compared to using OR statements. We're using UNION ALL since the queries
+    # used won't produce any duplicates (e.g. a note for a commit can't also be
+    # a note for an MR).
+    union = Gitlab::SQL::Union
+      .new([notes, commit_notes], remove_duplicates: false)
+      .to_sql
+
+    Note.from("(#{union}) #{Note.table_name}")
   end
 
   alias_method :discussion_notes, :related_notes
@@ -570,24 +598,6 @@ class MergeRequest < ActiveRecord::Base
     !discussions_to_be_resolved?
   end
 
-  def hook_attrs
-    attrs = {
-      source: source_project.try(:hook_attrs),
-      target: target_project.hook_attrs,
-      last_commit: nil,
-      work_in_progress: work_in_progress?,
-      total_time_spent: total_time_spent,
-      human_total_time_spent: human_total_time_spent,
-      human_time_estimate: human_time_estimate
-    }
-
-    if diff_head_commit
-      attrs[:last_commit] = diff_head_commit.hook_attrs
-    end
-
-    attributes.merge!(attrs)
-  end
-
   def for_fork?
     target_project != source_project
   end
@@ -672,13 +682,13 @@ class MergeRequest < ActiveRecord::Base
   def source_branch_exists?
     return false unless self.source_project
 
-    self.source_project.repository.branch_names.include?(self.source_branch)
+    self.source_project.repository.branch_exists?(self.source_branch)
   end
 
   def target_branch_exists?
     return false unless self.target_project
 
-    self.target_project.repository.branch_names.include?(self.target_branch)
+    self.target_project.repository.branch_exists?(self.target_branch)
   end
 
   def merge_commit_message(include_description: false)
@@ -734,10 +744,9 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def has_ci?
-    has_ci_integration = source_project.try(:ci_service)
-    uses_gitlab_ci = all_pipelines.any?
+    return false if has_no_commits?
 
-    (has_ci_integration || uses_gitlab_ci) && commits.any?
+    !!(head_pipeline_id || all_pipelines.any? || source_project&.ci_service)
   end
 
   def branch_missing?
@@ -872,7 +881,7 @@ class MergeRequest < ActiveRecord::Base
   #
   def all_commit_shas
     if persisted?
-      column_shas = MergeRequestDiffCommit.where(merge_request_diff: merge_request_diffs).pluck('DISTINCT(sha)')
+      column_shas = MergeRequestDiffCommit.where(merge_request_diff: merge_request_diffs).limit(10_000).pluck('sha')
       serialised_shas = merge_request_diffs.where.not(st_commits: nil).flat_map(&:commit_shas)
 
       (column_shas + serialised_shas).uniq
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index 58050e1f438..1eda0f9cbbd 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -48,6 +48,10 @@ class MergeRequestDiff < ActiveRecord::Base
   # Collect information about commits and diff from repository
   # and save it to the database as serialized data
   def save_git_content
+    MergeRequest
+      .where('id = ? AND COALESCE(latest_merge_request_diff_id, 0) < ?', self.merge_request_id, self.id)
+      .update_all(latest_merge_request_diff_id: self.id)
+
     ensure_commit_shas
     save_commits
     save_diffs
@@ -55,7 +59,6 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def ensure_commit_shas
-    merge_request.fetch_ref
     self.start_commit_sha ||= merge_request.target_branch_sha
     self.head_commit_sha  ||= merge_request.source_branch_sha
     self.base_commit_sha  ||= find_base_sha
diff --git a/app/models/merge_request_diff_commit.rb b/app/models/merge_request_diff_commit.rb
index 670b26d4ca3..b75387e236e 100644
--- a/app/models/merge_request_diff_commit.rb
+++ b/app/models/merge_request_diff_commit.rb
@@ -17,7 +17,9 @@ class MergeRequestDiffCommit < ActiveRecord::Base
       commit_hash.merge(
         merge_request_diff_id: merge_request_diff_id,
         relative_order: index,
-        sha: sha_attribute.type_cast_for_database(sha)
+        sha: sha_attribute.type_cast_for_database(sha),
+        authored_date: Gitlab::Database.sanitize_timestamp(commit_hash[:authored_date]),
+        committed_date: Gitlab::Database.sanitize_timestamp(commit_hash[:committed_date])
       )
     end
 
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index e279d8dd8c5..0601a61a926 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -139,7 +139,9 @@ class Namespace < ActiveRecord::Base
   end
 
   def find_fork_of(project)
-    projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id)
+    return nil unless project.fork_network
+
+    project.fork_network.find_forks_in(projects).first
   end
 
   def lfs_enabled?
@@ -160,6 +162,13 @@ class Namespace < ActiveRecord::Base
       .base_and_ancestors
   end
 
+  # returns all ancestors upto but excluding the the given namespace
+  # when no namespace is given, all ancestors upto the top are returned
+  def ancestors_upto(top = nil)
+    Gitlab::GroupHierarchy.new(self.class.where(id: id))
+      .ancestors(upto: top)
+  end
+
   def self_and_ancestors
     return self.class.where(id: id) unless parent_id
 
diff --git a/app/models/note.rb b/app/models/note.rb
index f44590e2144..f9676361072 100644
--- a/app/models/note.rb
+++ b/app/models/note.rb
@@ -69,7 +69,7 @@ class Note < ActiveRecord::Base
   delegate :title, to: :noteable, allow_nil: true
 
   validates :note, presence: true
-  validates :project, presence: true, unless: :for_personal_snippet?
+  validates :project, presence: true, if: :for_project_noteable?
 
   # Attachments are deprecated and are handled by Markdown uploader
   validates :attachment, file_size: { maximum: :max_attachment_size }
@@ -114,7 +114,7 @@ class Note < ActiveRecord::Base
   after_initialize :ensure_discussion_id
   before_validation :nullify_blank_type, :nullify_blank_line_code
   before_validation :set_discussion_id, on: :create
-  after_save :keep_around_commit, unless: :for_personal_snippet?
+  after_save :keep_around_commit, if: :for_project_noteable?
   after_save :expire_etag_cache
   after_destroy :expire_etag_cache
 
@@ -134,14 +134,22 @@ class Note < ActiveRecord::Base
       Discussion.build(notes)
     end
 
+    # Group diff discussions by line code or file path.
+    # It is not needed to group by line code when comment is
+    # on an image.
     def grouped_diff_discussions(diff_refs = nil)
       groups = {}
 
       diff_notes.fresh.discussions.each do |discussion|
-        line_code = discussion.line_code_in_diffs(diff_refs)
-
-        if line_code
-          discussions = groups[line_code] ||= []
+        group_key =
+          if discussion.on_image?
+            discussion.file_new_path
+          else
+            discussion.line_code_in_diffs(diff_refs)
+          end
+
+        if group_key
+          discussions = groups[group_key] ||= []
           discussions << discussion
         end
       end
@@ -161,7 +169,7 @@ class Note < ActiveRecord::Base
   end
 
   def cross_reference?
-    system? && SystemNoteService.cross_reference?(note)
+    system? && matches_cross_reference_regex?
   end
 
   def diff_note?
@@ -200,6 +208,10 @@ class Note < ActiveRecord::Base
     noteable.is_a?(PersonalSnippet)
   end
 
+  def for_project_noteable?
+    !for_personal_snippet?
+  end
+
   def skip_project_check?
     for_personal_snippet?
   end
diff --git a/app/models/oauth_access_token.rb b/app/models/oauth_access_token.rb
index b85f5dbaf2e..e8595b13d6d 100644
--- a/app/models/oauth_access_token.rb
+++ b/app/models/oauth_access_token.rb
@@ -1,4 +1,14 @@
 class OauthAccessToken < Doorkeeper::AccessToken
   belongs_to :resource_owner, class_name: 'User'
   belongs_to :application, class_name: 'Doorkeeper::Application'
+
+  alias_attribute :user, :resource_owner
+
+  def scopes=(value)
+    if value.is_a?(Array)
+      super(Doorkeeper::OAuth::Scopes.from_array(value).to_s)
+    else
+      super
+    end
+  end
 end
diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb
index 5d798247863..2e824cda525 100644
--- a/app/models/pages_domain.rb
+++ b/app/models/pages_domain.rb
@@ -16,9 +16,9 @@ class PagesDomain < ActiveRecord::Base
     key: Gitlab::Application.secrets.db_key_base,
     algorithm: 'aes-256-cbc'
 
-  after_create :update
-  after_save :update
-  after_destroy :update
+  after_create :update_daemon
+  after_save :update_daemon
+  after_destroy :update_daemon
 
   def to_param
     domain
@@ -80,7 +80,7 @@ class PagesDomain < ActiveRecord::Base
 
   private
 
-  def update
+  def update_daemon
     ::Projects::UpdatePagesConfigurationService.new(project).execute
   end
 
diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb
index 1f9d712ef84..cfcb03138b7 100644
--- a/app/models/personal_access_token.rb
+++ b/app/models/personal_access_token.rb
@@ -17,6 +17,8 @@ class PersonalAccessToken < ActiveRecord::Base
   validates :scopes, presence: true
   validate :validate_scopes
 
+  after_initialize :set_default_scopes, if: :persisted?
+
   def revoke!
     update!(revoked: true)
   end
@@ -32,4 +34,8 @@ class PersonalAccessToken < ActiveRecord::Base
       errors.add :scopes, "can only contain available scopes"
     end
   end
+
+  def set_default_scopes
+    self.scopes = Gitlab::Auth::DEFAULT_SCOPES if self.scopes.empty?
+  end
 end
diff --git a/app/models/project.rb b/app/models/project.rb
index f7221e4f3b2..3f810ee977b 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -17,6 +17,7 @@ class Project < ActiveRecord::Base
   include ProjectFeaturesCompatibility
   include SelectForProjectAuthorization
   include Routable
+  include GroupDescendant
 
   extend Gitlab::ConfigHelper
   extend Gitlab::CurrentSettings
@@ -25,7 +26,15 @@ class Project < ActiveRecord::Base
 
   NUMBER_OF_PERMITTED_BOARDS = 1
   UNKNOWN_IMPORT_URL = 'http://unknown.git'.freeze
-  LATEST_STORAGE_VERSION = 1
+  # Hashed Storage versions handle rolling out new storage to project and dependents models:
+  # nil: legacy
+  # 1: repository
+  # 2: attachments
+  LATEST_STORAGE_VERSION = 2
+  HASHED_STORAGE_FEATURES = {
+    repository: 1,
+    attachments: 2
+  }.freeze
 
   cache_markdown_field :description, pipeline: :description
 
@@ -64,6 +73,7 @@ class Project < ActiveRecord::Base
 
   # Storage specific hooks
   after_initialize :use_hashed_storage
+  after_create :check_repository_absence!
   after_create :ensure_storage_path_exists
   after_save :ensure_storage_path_exists, if: :namespace_id_changed?
 
@@ -72,6 +82,7 @@ class Project < ActiveRecord::Base
   attr_accessor :old_path_with_namespace
   attr_accessor :template_name
   attr_writer :pipeline_status
+  attr_accessor :skip_disk_validation
 
   alias_attribute :title, :name
 
@@ -79,6 +90,8 @@ class Project < ActiveRecord::Base
   belongs_to :creator, class_name: 'User'
   belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id'
   belongs_to :namespace
+  alias_method :parent, :namespace
+  alias_attribute :parent_id, :namespace_id
 
   has_one :last_event, -> {order 'events.created_at DESC'}, class_name: 'Event'
   has_many :boards, before_add: :validate_board_limit
@@ -115,12 +128,22 @@ class Project < ActiveRecord::Base
   has_one :mock_deployment_service
   has_one :mock_monitoring_service
   has_one :microsoft_teams_service
+  has_one :packagist_service
 
+  # TODO: replace these relations with the fork network versions
   has_one  :forked_project_link,  foreign_key: "forked_to_project_id"
   has_one  :forked_from_project,  through:   :forked_project_link
 
   has_many :forked_project_links, foreign_key: "forked_from_project_id"
   has_many :forks,                through:     :forked_project_links, source: :forked_to_project
+  # TODO: replace these relations with the fork network versions
+
+  has_one :root_of_fork_network,
+          foreign_key: 'root_project_id',
+          inverse_of: :root_project,
+          class_name: 'ForkNetwork'
+  has_one :fork_network_member
+  has_one :fork_network, through: :fork_network_member
 
   # Merge Requests for target project should be removed with it
   has_many :merge_requests, foreign_key: 'target_project_id'
@@ -163,6 +186,7 @@ class Project < ActiveRecord::Base
   has_one :import_data, class_name: 'ProjectImportData', inverse_of: :project, autosave: true
   has_one :project_feature, inverse_of: :project
   has_one :statistics, class_name: 'ProjectStatistics'
+  has_one :cluster, class_name: 'Gcp::Cluster', inverse_of: :project
 
   # Container repositories need to remove data from the container registry,
   # which is not managed by the DB. Hence we're still using dependent: :destroy
@@ -177,6 +201,7 @@ class Project < ActiveRecord::Base
   # bulk that doesn't involve loading the rows into memory. As a result we're
   # still using `dependent: :destroy` here.
   has_many :builds, class_name: 'Ci::Build', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+  has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName'
   has_many :runner_projects, class_name: 'Ci::RunnerProject'
   has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
   has_many :variables, class_name: 'Ci::Variable'
@@ -227,7 +252,7 @@ class Project < ActiveRecord::Base
   validates :import_url, importable_url: true, if: [:external_import?, :import_url_changed?]
   validates :star_count, numericality: { greater_than_or_equal_to: 0 }
   validate :check_limit, on: :create
-  validate :can_create_repository?, on: [:create, :update], if: ->(project) { !project.persisted? || project.renamed? }
+  validate :check_repository_path_availability, on: :update, if: ->(project) { project.renamed? }
   validate :avatar_type,
     if: ->(project) { project.avatar.present? && project.avatar_changed? }
   validates :avatar, file_size: { maximum: 200.kilobytes.to_i }
@@ -245,6 +270,9 @@ class Project < ActiveRecord::Base
   scope :pending_delete, -> { where(pending_delete: true) }
   scope :without_deleted, -> { where(pending_delete: false) }
 
+  scope :with_hashed_storage, -> { where('storage_version >= 1') }
+  scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) }
+
   scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) }
   scope :sorted_by_stars, -> { reorder('projects.star_count DESC') }
 
@@ -463,6 +491,13 @@ class Project < ActiveRecord::Base
     end
   end
 
+  # returns all ancestor-groups upto but excluding the given namespace
+  # when no namespace is given, all ancestors upto the top are returned
+  def ancestors_upto(top = nil)
+    Gitlab::GroupHierarchy.new(Group.where(id: namespace_id))
+      .base_and_ancestors(upto: top)
+  end
+
   def lfs_enabled?
     return namespace.lfs_enabled? if self[:lfs_enabled].nil?
 
@@ -514,6 +549,10 @@ class Project < ActiveRecord::Base
     repository.commit(ref)
   end
 
+  def commit_by(oid:)
+    repository.commit_by(oid: oid)
+  end
+
   # ref can't be HEAD, can only be branch/tag name or SHA
   def latest_successful_builds_for(ref = default_branch)
     latest_pipeline = pipelines.latest_successful_for(ref)
@@ -527,7 +566,7 @@ class Project < ActiveRecord::Base
 
   def merge_base_commit(first_commit_id, second_commit_id)
     sha = repository.merge_base(first_commit_id, second_commit_id)
-    repository.commit(sha) if sha
+    commit_by(oid: sha) if sha
   end
 
   def saved?
@@ -808,7 +847,7 @@ class Project < ActiveRecord::Base
   end
 
   def cache_has_external_issue_tracker
-    update_column(:has_external_issue_tracker, services.external_issue_trackers.any?)
+    update_column(:has_external_issue_tracker, services.external_issue_trackers.any?) if Gitlab::Database.read_write?
   end
 
   def has_wiki?
@@ -828,7 +867,7 @@ class Project < ActiveRecord::Base
   end
 
   def cache_has_external_wiki
-    update_column(:has_external_wiki, services.external_wikis.any?)
+    update_column(:has_external_wiki, services.external_wikis.any?) if Gitlab::Database.read_write?
   end
 
   def find_or_initialize_services(exceptions: [])
@@ -993,9 +1032,18 @@ class Project < ActiveRecord::Base
   end
 
   def forked?
+    return true if fork_network && fork_network.root_project != self
+
+    # TODO: Use only the above conditional using the `fork_network`
+    # This is the old conditional that looks at the `forked_project_link`, we
+    # fall back to this while we're migrating the new models
     !(forked_project_link.nil? || forked_project_link.forked_from_project.nil?)
   end
 
+  def fork_source
+    forked_from_project || fork_network&.root_project
+  end
+
   def personal?
     !group
   end
@@ -1015,24 +1063,29 @@ class Project < ActiveRecord::Base
   end
 
   # Check if repository already exists on disk
-  def can_create_repository?
+  def check_repository_path_availability
+    return true if skip_disk_validation
     return false unless repository_storage_path
 
     expires_full_path_cache # we need to clear cache to validate renames correctly
 
-    if gitlab_shell.exists?(repository_storage_path, "#{disk_path}.git")
+    # Check if repository with same path already exists on disk we can
+    # skip this for the hashed storage because the path does not change
+    if legacy_storage? && repository_with_same_path_already_exists?
       errors.add(:base, 'There is already a repository with that name on disk')
       return false
     end
 
     true
+  rescue GRPC::Internal # if the path is too long
+    false
   end
 
   def create_repository(force: false)
     # Forked import is handled asynchronously
     return if forked? && !force
 
-    if gitlab_shell.add_repository(repository_storage_path, disk_path)
+    if gitlab_shell.add_repository(repository_storage, disk_path)
       repository.after_create
       true
     else
@@ -1043,6 +1096,7 @@ class Project < ActiveRecord::Base
 
   def hook_attrs(backward: true)
     attrs = {
+      id: id,
       name: name,
       description: description,
       web_url: web_url,
@@ -1107,8 +1161,19 @@ class Project < ActiveRecord::Base
     end
   end
 
-  def forked_from?(project)
-    forked? && project == forked_from_project
+  def forked_from?(other_project)
+    forked? && forked_from_project == other_project
+  end
+
+  def in_fork_network_of?(other_project)
+    # TODO: Remove this in a next release when all fork_networks are populated
+    # This makes sure all MergeRequests remain valid while the projects don't
+    # have a fork_network yet.
+    return true if forked_from?(other_project)
+
+    return false if fork_network.nil? || other_project.fork_network.nil?
+
+    fork_network == other_project.fork_network
   end
 
   def origin_merge_requests
@@ -1225,7 +1290,7 @@ class Project < ActiveRecord::Base
 
     # self.forked_from_project will be nil before the project is saved, so
     # we need to go through the relation
-    original_project = forked_project_link.forked_from_project
+    original_project = forked_project_link&.forked_from_project
     return true unless original_project
 
     level <= original_project.visibility_level
@@ -1343,6 +1408,19 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def after_rename_repo
+    path_before_change = previous_changes['path'].first
+
+    # We need to check if project had been rolled out to move resource to hashed storage or not and decide
+    # if we need execute any take action or no-op.
+
+    unless hashed_storage?(:attachments)
+      Gitlab::UploadsTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
+    end
+
+    Gitlab::PagesTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
+  end
+
   def rename_repo_notify!
     send_move_instructions(full_path_was)
     expires_full_path_cache
@@ -1353,13 +1431,6 @@ class Project < ActiveRecord::Base
     reload_repository!
   end
 
-  def after_rename_repo
-    path_before_change = previous_changes['path'].first
-
-    Gitlab::UploadsTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
-    Gitlab::PagesTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
-  end
-
   def running_or_pending_build_count(force: false)
     Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do
       builds.running_or_pending.count(:all)
@@ -1421,7 +1492,8 @@ class Project < ActiveRecord::Base
       { key: 'CI_PROJECT_PATH', value: full_path, public: true },
       { key: 'CI_PROJECT_PATH_SLUG', value: full_path_slug, public: true },
       { key: 'CI_PROJECT_NAMESPACE', value: namespace.full_path, public: true },
-      { key: 'CI_PROJECT_URL', value: web_url, public: true }
+      { key: 'CI_PROJECT_URL', value: web_url, public: true },
+      { key: 'CI_PROJECT_VISIBILITY', value: Gitlab::VisibilityLevel.string_level(visibility_level), public: true }
     ]
   end
 
@@ -1512,10 +1584,6 @@ class Project < ActiveRecord::Base
     map.public_path_for_source_path(path)
   end
 
-  def parent
-    namespace
-  end
-
   def parent_changed?
     namespace_id_changed?
   end
@@ -1550,18 +1618,81 @@ class Project < ActiveRecord::Base
   end
 
   def legacy_storage?
-    self.storage_version.nil?
+    [nil, 0].include?(self.storage_version)
+  end
+
+  # Check if Hashed Storage is enabled for the project with at least informed feature rolled out
+  #
+  # @param [Symbol] feature that needs to be rolled out for the project (:repository, :attachments)
+  def hashed_storage?(feature)
+    raise ArgumentError, "Invalid feature" unless HASHED_STORAGE_FEATURES.include?(feature)
+
+    self.storage_version && self.storage_version >= HASHED_STORAGE_FEATURES[feature]
   end
 
   def renamed?
     persisted? && path_changed?
   end
 
+  def merge_method
+    if self.merge_requests_ff_only_enabled
+      :ff
+    elsif self.merge_requests_rebase_enabled
+      :rebase_merge
+    else
+      :merge
+    end
+  end
+
+  def merge_method=(method)
+    case method.to_s
+    when "ff"
+      self.merge_requests_ff_only_enabled = true
+      self.merge_requests_rebase_enabled = true
+    when "rebase_merge"
+      self.merge_requests_ff_only_enabled = false
+      self.merge_requests_rebase_enabled = true
+    when "merge"
+      self.merge_requests_ff_only_enabled = false
+      self.merge_requests_rebase_enabled = false
+    end
+  end
+
+  def ff_merge_must_be_possible?
+    self.merge_requests_ff_only_enabled || self.merge_requests_rebase_enabled
+  end
+
+  def migrate_to_hashed_storage!
+    return if hashed_storage?(:repository)
+
+    update!(repository_read_only: true)
+
+    if repo_reference_count > 0 || wiki_reference_count > 0
+      ProjectMigrateHashedStorageWorker.perform_in(Gitlab::ReferenceCounter::REFERENCE_EXPIRE_TIME, id)
+    else
+      ProjectMigrateHashedStorageWorker.perform_async(id)
+    end
+  end
+
+  def storage_version=(value)
+    super
+
+    @storage = nil if storage_version_changed?
+  end
+
+  def gl_repository(is_wiki:)
+    Gitlab::GlRepository.gl_repository(self, is_wiki)
+  end
+
+  def reference_counter(wiki: false)
+    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: wiki))
+  end
+
   private
 
   def storage
     @storage ||=
-      if self.storage_version && self.storage_version >= 1
+      if hashed_storage?(:repository)
         Storage::HashedProject.new(self)
       else
         Storage::LegacyProject.new(self)
@@ -1574,6 +1705,27 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def repo_reference_count
+    reference_counter.value
+  end
+
+  def wiki_reference_count
+    reference_counter(wiki: true).value
+  end
+
+  def check_repository_absence!
+    return if skip_disk_validation
+
+    if repository_storage_path.blank? || repository_with_same_path_already_exists?
+      errors.add(:base, 'There is already a repository with that name on disk')
+      throw :abort
+    end
+  end
+
+  def repository_with_same_path_already_exists?
+    gitlab_shell.exists?(repository_storage_path, "#{disk_path}.git")
+  end
+
   # set last_activity_at to the same as created_at
   def set_last_activity_at
     update_column(:last_activity_at, self.created_at)
diff --git a/app/models/project_services/chat_message/base_message.rb b/app/models/project_services/chat_message/base_message.rb
index e2ad586aea7..22a65b5145e 100644
--- a/app/models/project_services/chat_message/base_message.rb
+++ b/app/models/project_services/chat_message/base_message.rb
@@ -3,6 +3,7 @@ require 'slack-notifier'
 module ChatMessage
   class BaseMessage
     attr_reader :markdown
+    attr_reader :user_full_name
     attr_reader :user_name
     attr_reader :user_avatar
     attr_reader :project_name
@@ -12,10 +13,19 @@ module ChatMessage
       @markdown = params[:markdown] || false
       @project_name = params.dig(:project, :path_with_namespace) || params[:project_name]
       @project_url = params.dig(:project, :web_url) || params[:project_url]
+      @user_full_name = params.dig(:user, :name) || params[:user_full_name]
       @user_name = params.dig(:user, :username) || params[:user_name]
       @user_avatar = params.dig(:user, :avatar_url) || params[:user_avatar]
     end
 
+    def user_combined_name
+      if user_full_name.present?
+        "#{user_full_name} (#{user_name})"
+      else
+        user_name
+      end
+    end
+
     def pretext
       return message if markdown
 
diff --git a/app/models/project_services/chat_message/issue_message.rb b/app/models/project_services/chat_message/issue_message.rb
index 4b9a2b1e1f3..1327b075858 100644
--- a/app/models/project_services/chat_message/issue_message.rb
+++ b/app/models/project_services/chat_message/issue_message.rb
@@ -29,7 +29,7 @@ module ChatMessage
 
     def activity
       {
-        title: "Issue #{state} by #{user_name}",
+        title: "Issue #{state} by #{user_combined_name}",
         subtitle: "in #{project_link}",
         text: issue_link,
         image: user_avatar
@@ -40,9 +40,9 @@ module ChatMessage
 
     def message
       if state == 'opened'
-        "[#{project_link}] Issue #{state} by #{user_name}"
+        "[#{project_link}] Issue #{state} by #{user_combined_name}"
       else
-        "[#{project_link}] Issue #{issue_link} #{state} by #{user_name}"
+        "[#{project_link}] Issue #{issue_link} #{state} by #{user_combined_name}"
       end
     end
 
diff --git a/app/models/project_services/chat_message/merge_message.rb b/app/models/project_services/chat_message/merge_message.rb
index 7d0de81cdf0..f412b6833d9 100644
--- a/app/models/project_services/chat_message/merge_message.rb
+++ b/app/models/project_services/chat_message/merge_message.rb
@@ -24,7 +24,7 @@ module ChatMessage
 
     def activity
       {
-        title: "Merge Request #{state} by #{user_name}",
+        title: "Merge Request #{state} by #{user_combined_name}",
         subtitle: "in #{project_link}",
         text: merge_request_link,
         image: user_avatar
@@ -46,7 +46,7 @@ module ChatMessage
     end
 
     def merge_request_message
-      "#{user_name} #{state} #{merge_request_link} in #{project_link}: #{title}"
+      "#{user_combined_name} #{state} #{merge_request_link} in #{project_link}: #{title}"
     end
 
     def merge_request_link
diff --git a/app/models/project_services/chat_message/note_message.rb b/app/models/project_services/chat_message/note_message.rb
index 2da4c244229..7f9486132e6 100644
--- a/app/models/project_services/chat_message/note_message.rb
+++ b/app/models/project_services/chat_message/note_message.rb
@@ -32,7 +32,7 @@ module ChatMessage
 
     def activity
       {
-        title: "#{user_name} #{link('commented on ' + target, note_url)}",
+        title: "#{user_combined_name} #{link('commented on ' + target, note_url)}",
         subtitle: "in #{project_link}",
         text: formatted_title,
         image: user_avatar
@@ -42,7 +42,7 @@ module ChatMessage
     private
 
     def message
-      "#{user_name} #{link('commented on ' + target, note_url)} in #{project_link}: *#{formatted_title}*"
+      "#{user_combined_name} #{link('commented on ' + target, note_url)} in #{project_link}: *#{formatted_title}*"
     end
 
     def format_title(title)
diff --git a/app/models/project_services/chat_message/pipeline_message.rb b/app/models/project_services/chat_message/pipeline_message.rb
index d63d4ec2b12..2135122278a 100644
--- a/app/models/project_services/chat_message/pipeline_message.rb
+++ b/app/models/project_services/chat_message/pipeline_message.rb
@@ -9,7 +9,7 @@ module ChatMessage
     def initialize(data)
       super
 
-      @user_name = data.dig(:user, :name) || 'API'
+      @user_name = data.dig(:user, :username) || 'API'
 
       pipeline_attributes = data[:object_attributes]
       @ref_type = pipeline_attributes[:tag] ? 'tag' : 'branch'
@@ -35,7 +35,7 @@ module ChatMessage
 
     def activity
       {
-        title: "Pipeline #{pipeline_link} of #{ref_type} #{branch_link} by #{user_name} #{humanized_status}",
+        title: "Pipeline #{pipeline_link} of #{ref_type} #{branch_link} by #{user_combined_name} #{humanized_status}",
         subtitle: "in #{project_link}",
         text: "in #{pretty_duration(duration)}",
         image: user_avatar || ''
@@ -45,7 +45,7 @@ module ChatMessage
     private
 
     def message
-      "#{project_link}: Pipeline #{pipeline_link} of #{ref_type} #{branch_link} by #{user_name} #{humanized_status} in #{pretty_duration(duration)}"
+      "#{project_link}: Pipeline #{pipeline_link} of #{ref_type} #{branch_link} by #{user_combined_name} #{humanized_status} in #{pretty_duration(duration)}"
     end
 
     def humanized_status
diff --git a/app/models/project_services/chat_message/push_message.rb b/app/models/project_services/chat_message/push_message.rb
index c52dd6ef8ef..8d599c5f116 100644
--- a/app/models/project_services/chat_message/push_message.rb
+++ b/app/models/project_services/chat_message/push_message.rb
@@ -33,7 +33,7 @@ module ChatMessage
                end
 
       {
-        title: "#{user_name} #{action} #{ref_type}",
+        title: "#{user_combined_name} #{action} #{ref_type}",
         subtitle: "in #{project_link}",
         text: compare_link,
         image: user_avatar
@@ -57,15 +57,15 @@ module ChatMessage
     end
 
     def new_branch_message
-      "#{user_name} pushed new #{ref_type} #{branch_link} to #{project_link}"
+      "#{user_combined_name} pushed new #{ref_type} #{branch_link} to #{project_link}"
     end
 
     def removed_branch_message
-      "#{user_name} removed #{ref_type} #{ref} from #{project_link}"
+      "#{user_combined_name} removed #{ref_type} #{ref} from #{project_link}"
     end
 
     def push_message
-      "#{user_name} pushed to #{ref_type} #{branch_link} of #{project_link} (#{compare_link})"
+      "#{user_combined_name} pushed to #{ref_type} #{branch_link} of #{project_link} (#{compare_link})"
     end
 
     def commit_messages
diff --git a/app/models/project_services/chat_message/wiki_page_message.rb b/app/models/project_services/chat_message/wiki_page_message.rb
index a139a8ee727..d84b80f2de2 100644
--- a/app/models/project_services/chat_message/wiki_page_message.rb
+++ b/app/models/project_services/chat_message/wiki_page_message.rb
@@ -31,7 +31,7 @@ module ChatMessage
 
     def activity
       {
-        title: "#{user_name} #{action} #{wiki_page_link}",
+        title: "#{user_combined_name} #{action} #{wiki_page_link}",
         subtitle: "in #{project_link}",
         text: title,
         image: user_avatar
@@ -41,7 +41,7 @@ module ChatMessage
     private
 
     def message
-      "#{user_name} #{action} #{wiki_page_link} in #{project_link}: *#{title}*"
+      "#{user_combined_name} #{action} #{wiki_page_link} in #{project_link}: *#{title}*"
     end
 
     def description_message
diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb
index 9ee3a533c1e..b487378edd2 100644
--- a/app/models/project_services/jira_service.rb
+++ b/app/models/project_services/jira_service.rb
@@ -3,6 +3,8 @@ class JiraService < IssueTrackerService
 
   validates :url, url: true, presence: true, if: :activated?
   validates :api_url, url: true, allow_blank: true
+  validates :username, presence: true, if: :activated?
+  validates :password, presence: true, if: :activated?
 
   prop_accessor :username, :password, :url, :api_url, :jira_issue_transition_id, :title, :description
 
diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb
index 8ba07173c74..5c0b3338a62 100644
--- a/app/models/project_services/kubernetes_service.rb
+++ b/app/models/project_services/kubernetes_service.rb
@@ -153,7 +153,10 @@ class KubernetesService < DeploymentService
   end
 
   def default_namespace
-    "#{project.path}-#{project.id}" if project.present?
+    return unless project
+
+    slug = "#{project.path}-#{project.id}".downcase
+    slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '')
   end
 
   def build_kubeclient!(api_path: 'api', api_version: 'v1')
diff --git a/app/models/project_services/packagist_service.rb b/app/models/project_services/packagist_service.rb
new file mode 100644
index 00000000000..f68a0c1a3c3
--- /dev/null
+++ b/app/models/project_services/packagist_service.rb
@@ -0,0 +1,65 @@
+class PackagistService < Service
+  include HTTParty
+
+  prop_accessor :username, :token, :server
+
+  validates :username, presence: true, if: :activated?
+  validates :token, presence: true, if: :activated?
+
+  default_value_for :push_events, true
+  default_value_for :tag_push_events, true
+
+  after_save :compose_service_hook, if: :activated?
+
+  def title
+    'Packagist'
+  end
+
+  def description
+    'Update your project on Packagist, the main Composer repository'
+  end
+
+  def self.to_param
+    'packagist'
+  end
+
+  def fields
+    [
+      { type: 'text', name: 'username', placeholder: '', required: true },
+      { type: 'text', name: 'token', placeholder: '', required: true },
+      { type: 'text', name: 'server', placeholder: 'https://packagist.org', required: false }
+    ]
+  end
+
+  def self.supported_events
+    %w(push merge_request tag_push)
+  end
+
+  def execute(data)
+    return unless supported_events.include?(data[:object_kind])
+
+    service_hook.execute(data)
+  end
+
+  def test(data)
+    begin
+      result = execute(data)
+      return { success: false, result: result[:message] } if result[:http_status] != 202
+    rescue StandardError => error
+      return { success: false, result: error }
+    end
+
+    { success: true, result: result[:message] }
+  end
+
+  def compose_service_hook
+    hook = service_hook || build_service_hook
+    hook.url = hook_url
+    hook.save
+  end
+
+  def hook_url
+    base_url = server.present? ? server : 'https://packagist.org'
+    "#{base_url}/api/update-package?username=#{username}&apiToken=#{token}"
+  end
+end
diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb
index 698fdf7a20c..43de6809178 100644
--- a/app/models/project_wiki.rb
+++ b/app/models/project_wiki.rb
@@ -54,12 +54,15 @@ class ProjectWiki
     [Gitlab.config.gitlab.relative_url_root, '/', @project.full_path, '/wikis'].join('')
   end
 
-  # Returns the Gollum::Wiki object.
+  # Returns the Gitlab::Git::Wiki object.
   def wiki
     @wiki ||= begin
-      Gollum::Wiki.new(path_to_repo)
-    rescue Rugged::OSError
-      create_repo!
+      gl_repository = Gitlab::GlRepository.gl_repository(project, true)
+      raw_repository = Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', gl_repository)
+
+      create_repo!(raw_repository) unless raw_repository.exists?
+
+      Gitlab::Git::Wiki.new(raw_repository)
     end
   end
 
@@ -86,20 +89,14 @@ class ProjectWiki
   # Returns an initialized WikiPage instance or nil
   def find_page(title, version = nil)
     page_title, page_dir = page_title_and_dir(title)
-    if page = wiki.page(page_title, version, page_dir)
+
+    if page = wiki.page(title: page_title, version: version, dir: page_dir)
       WikiPage.new(self, page, true)
-    else
-      nil
     end
   end
 
-  def find_file(name, version = nil, try_on_disk = true)
-    version = wiki.ref if version.nil? # Gollum::Wiki#file ?
-    if wiki_file = wiki.file(name, version, try_on_disk)
-      wiki_file
-    else
-      nil
-    end
+  def find_file(name, version = nil)
+    wiki.file(name, version)
   end
 
   def create_page(title, content, format = :markdown, message = nil)
@@ -108,7 +105,7 @@ class ProjectWiki
     wiki.write_page(title, format.to_sym, content, commit)
 
     update_project_activity
-  rescue Gollum::DuplicatePageError => e
+  rescue Gitlab::Git::Wiki::DuplicatePageError => e
     @error_message = "Duplicate page: #{e.message}"
     return false
   end
@@ -116,13 +113,13 @@ class ProjectWiki
   def update_page(page, content:, title: nil, format: :markdown, message: nil)
     commit = commit_details(:updated, message, page.title)
 
-    wiki.update_page(page, title || page.name, format.to_sym, content, commit)
+    wiki.update_page(page.path, title || page.name, format.to_sym, content, commit)
 
     update_project_activity
   end
 
   def delete_page(page, message = nil)
-    wiki.delete_page(page, commit_details(:deleted, message, page.title))
+    wiki.delete_page(page.path, commit_details(:deleted, message, page.title))
 
     update_project_activity
   end
@@ -138,27 +135,15 @@ class ProjectWiki
   end
 
   def repository
-    @repository ||= Repository.new(full_path, @project, disk_path: disk_path)
+    @repository ||= Repository.new(full_path, @project, disk_path: disk_path, is_wiki: true)
   end
 
   def default_branch
     wiki.class.default_ref
   end
 
-  def create_repo!
-    if init_repo(disk_path)
-      wiki = Gollum::Wiki.new(path_to_repo)
-    else
-      raise CouldNotCreateWikiError
-    end
-
-    repository.after_create
-
-    wiki
-  end
-
   def ensure_repository
-    create_repo! unless repository_exists?
+    raise CouldNotCreateWikiError unless wiki.repository_exists?
   end
 
   def hook_attrs
@@ -173,24 +158,24 @@ class ProjectWiki
 
   private
 
-  def init_repo(disk_path)
-    gitlab_shell.add_repository(project.repository_storage_path, disk_path)
+  def create_repo!(raw_repository)
+    gitlab_shell.add_repository(project.repository_storage, disk_path)
+
+    raise CouldNotCreateWikiError unless raw_repository.exists?
+
+    repository.after_create
   end
 
   def commit_details(action, message = nil, title = nil)
     commit_message = message || default_message(action, title)
 
-    { email: @user.email, name: @user.name, message: commit_message }
+    Gitlab::Git::Wiki::CommitDetails.new(@user.name, @user.email, commit_message)
   end
 
   def default_message(action, title)
     "#{@user.username} #{action} page: #{title}"
   end
 
-  def path_to_repo
-    @path_to_repo ||= File.join(project.repository_storage_path, "#{disk_path}.git")
-  end
-
   def update_project_activity
     @project.touch(:last_activity_at, :last_repository_updated_at)
   end
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 90cede9d3d4..69cddb36b2e 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -15,9 +15,8 @@ class Repository
   ].freeze
 
   include Gitlab::ShellAdapter
-  include RepositoryMirroring
 
-  attr_accessor :full_path, :disk_path, :project
+  attr_accessor :full_path, :disk_path, :project, :is_wiki
 
   delegate :ref_name_for_sha, to: :raw_repository
 
@@ -34,7 +33,11 @@ class Repository
   CACHED_METHODS = %i(size commit_count rendered_readme contribution_guide
                       changelog license_blob license_key gitignore koding_yml
                       gitlab_ci_yml branch_names tag_names branch_count
-                      tag_count avatar exists? empty? root_ref).freeze
+                      tag_count avatar exists? empty? root_ref has_visible_content?
+                      issue_template_names merge_request_template_names).freeze
+
+  # Methods that use cache_method but only memoize the value
+  MEMOIZED_CACHED_METHODS = %i(license empty_repo?).freeze
 
   # Certain method caches should be refreshed when certain types of files are
   # changed. This Hash maps file types (as returned by Gitlab::FileDetector) to
@@ -47,7 +50,9 @@ class Repository
     gitignore: :gitignore,
     koding: :koding_yml,
     gitlab_ci: :gitlab_ci_yml,
-    avatar: :avatar
+    avatar: :avatar,
+    issue_template: :issue_template_names,
+    merge_request_template: :merge_request_template_names
   }.freeze
 
   # Wraps around the given method and caches its output in Redis and an instance
@@ -66,10 +71,12 @@ class Repository
     end
   end
 
-  def initialize(full_path, project, disk_path: nil)
+  def initialize(full_path, project, disk_path: nil, is_wiki: false)
     @full_path = full_path
     @disk_path = disk_path || full_path
     @project = project
+    @commit_cache = {}
+    @is_wiki = is_wiki
   end
 
   def ==(other)
@@ -91,30 +98,23 @@ class Repository
     )
   end
 
-  # we need to have this method here because it is not cached in ::Git and
-  # the method is called multiple times for every request
-  def has_visible_content?
-    branch_count > 0
-  end
-
   def inspect
     "#<#{self.class.name}:#{@disk_path}>"
   end
 
   def commit(ref = 'HEAD')
     return nil unless exists?
+    return ref if ref.is_a?(::Commit)
 
-    commit =
-      if ref.is_a?(Gitlab::Git::Commit)
-        ref
-      else
-        Gitlab::Git::Commit.find(raw_repository, ref)
-      end
+    find_commit(ref)
+  end
 
-    commit = ::Commit.new(commit, @project) if commit
-    commit
-  rescue Rugged::OdbError, Rugged::TreeError
-    nil
+  # Finding a commit by the passed SHA
+  # Also takes care of caching, based on the SHA
+  def commit_by(oid:)
+    return @commit_cache[oid] if @commit_cache.key?(oid)
+
+    @commit_cache[oid] = find_commit(oid)
   end
 
   def commits(ref, path: nil, limit: nil, offset: nil, skip_merges: false, after: nil, before: nil)
@@ -231,7 +231,7 @@ class Repository
   # branches or tags, but we want to keep some of these commits around, for
   # example if they have comments or CI builds.
   def keep_around(sha)
-    return unless sha && commit(sha)
+    return unless sha && commit_by(oid: sha)
 
     return if kept_around?(sha)
 
@@ -275,7 +275,7 @@ class Repository
   end
 
   def expire_branches_cache
-    expire_method_caches(%i(branch_names branch_count))
+    expire_method_caches(%i(branch_names branch_count has_visible_content?))
     @local_branches = nil
     @branch_exists_memo = nil
   end
@@ -346,7 +346,7 @@ class Repository
   def expire_emptiness_caches
     return unless empty?
 
-    expire_method_caches(%i(empty?))
+    expire_method_caches(%i(empty? has_visible_content?))
   end
 
   def lookup_cache
@@ -468,9 +468,7 @@ class Repository
   end
 
   def blob_at(sha, path)
-    unless Gitlab::Git.blank_ref?(sha)
-      Blob.decorate(Gitlab::Git::Blob.find(self, sha, path), project)
-    end
+    Blob.decorate(raw_repository.blob_at(sha, path), project)
   rescue Gitlab::Git::Repository::NoRepository
     nil
   end
@@ -489,13 +487,7 @@ class Repository
   def exists?
     return false unless full_path
 
-    Gitlab::GitalyClient.migrate(:repository_exists) do |enabled|
-      if enabled
-        raw_repository.exists?
-      else
-        refs_directory_exists?
-      end
-    end
+    raw_repository.exists?
   end
   cache_method :exists?
 
@@ -529,17 +521,31 @@ class Repository
   delegate :tag_names, to: :raw_repository
   cache_method :tag_names, fallback: []
 
-  delegate :branch_count, :tag_count, to: :raw_repository
+  delegate :branch_count, :tag_count, :has_visible_content?, to: :raw_repository
   cache_method :branch_count, fallback: 0
   cache_method :tag_count, fallback: 0
+  cache_method :has_visible_content?, fallback: false
 
   def avatar
-    if tree = file_on_head(:avatar)
-      tree.path
+    # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/38327
+    Gitlab::GitalyClient.allow_n_plus_1_calls do
+      if tree = file_on_head(:avatar)
+        tree.path
+      end
     end
   end
   cache_method :avatar
 
+  def issue_template_names
+    Gitlab::Template::IssueTemplate.dropdown_names(project)
+  end
+  cache_method :issue_template_names, fallback: []
+
+  def merge_request_template_names
+    Gitlab::Template::MergeRequestTemplate.dropdown_names(project)
+  end
+  cache_method :merge_request_template_names, fallback: []
+
   def readme
     if readme = tree(:head)&.readme
       ReadmeBlob.new(readme, self)
@@ -855,6 +861,15 @@ class Repository
     end
   end
 
+  def ff_merge(user, source, target_branch, merge_request: nil)
+    their_commit_id = commit(source)&.id
+    raise 'Invalid merge source' if their_commit_id.nil?
+
+    merge_request&.update(in_progress_merge_commit_sha: their_commit_id)
+
+    with_cache_hooks { raw.ff_merge(user, their_commit_id, target_branch) }
+  end
+
   def revert(
     user, commit, branch_name, message,
     start_branch_name: nil, start_project: project)
@@ -887,26 +902,27 @@ class Repository
     end
   end
 
-  def resolve_conflicts(user, branch_name, params)
-    with_branch(user, branch_name) do
-      committer = user_to_committer(user)
+  def merged_to_root_ref?(branch_or_name, pre_loaded_merged_branches = nil)
+    branch = Gitlab::Git::Branch.find(self, branch_or_name)
 
-      create_commit(params.merge(author: committer, committer: committer))
-    end
-  end
-
-  def merged_to_root_ref?(branch_name)
-    branch_commit = commit(branch_name)
-    root_ref_commit = commit(root_ref)
+    if branch
+      root_ref_sha = commit(root_ref).sha
+      same_head = branch.target == root_ref_sha
+      merged =
+        if pre_loaded_merged_branches
+          pre_loaded_merged_branches.include?(branch.name)
+        else
+          ancestor?(branch.target, root_ref_sha)
+        end
 
-    if branch_commit
-      same_head = branch_commit.id == root_ref_commit.id
-      !same_head && ancestor?(branch_commit.id, root_ref_commit.id)
+      !same_head && merged
     else
       nil
     end
   end
 
+  delegate :merged_branch_names, to: :raw_repository
+
   def merge_base(first_commit_id, second_commit_id)
     first_commit_id = commit(first_commit_id).try(:id) || first_commit_id
     second_commit_id = commit(second_commit_id).try(:id) || second_commit_id
@@ -949,21 +965,8 @@ class Repository
     run_git(args).first.lines.map(&:strip)
   end
 
-  def add_remote(name, url)
-    raw_repository.remote_add(name, url)
-  rescue Rugged::ConfigError
-    raw_repository.remote_update(name, url: url)
-  end
-
-  def remove_remote(name)
-    raw_repository.remote_delete(name)
-    true
-  rescue Rugged::ConfigError
-    false
-  end
-
-  def fetch_remote(remote, forced: false, no_tags: false)
-    gitlab_shell.fetch_remote(raw_repository, remote, forced: forced, no_tags: no_tags)
+  def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false)
+    gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
   end
 
   def fetch_source_branch(source_repository, source_branch, local_ref)
@@ -975,7 +978,7 @@ class Repository
   end
 
   def create_ref(ref, ref_path)
-    fetch_ref(path_to_repo, ref, ref_path)
+    raw_repository.write_ref(ref_path, ref)
   end
 
   def ls_files(ref)
@@ -1014,6 +1017,10 @@ class Repository
     if instance_variable_defined?(ivar)
       instance_variable_get(ivar)
     else
+      # If the repository doesn't exist and a fallback was specified we return
+      # that value inmediately. This saves us Rugged/gRPC invocations.
+      return fallback unless fallback.nil? || exists?
+
       begin
         value =
           if memoize_only
@@ -1023,8 +1030,9 @@ class Repository
           end
         instance_variable_set(ivar, value)
       rescue Rugged::ReferenceError, Gitlab::Git::Repository::NoRepository
-        # if e.g. HEAD or the entire repository doesn't exist we want to
-        # gracefully handle this and not cache anything.
+        # Even if the above `#exists?` check passes these errors might still
+        # occur (for example because of a non-existing HEAD). We want to
+        # gracefully handle this and not cache anything
         fallback
       end
     end
@@ -1052,6 +1060,18 @@ class Repository
 
   private
 
+  # TODO Generice finder, later split this on finders by Ref or Oid
+  # gitlab-org/gitlab-ce#39239
+  def find_commit(oid_or_ref)
+    commit = if oid_or_ref.is_a?(Gitlab::Git::Commit)
+               oid_or_ref
+             else
+               Gitlab::Git::Commit.find(raw_repository, oid_or_ref)
+             end
+
+    ::Commit.new(commit, @project) if commit
+  end
+
   def blob_data_at(sha, path)
     blob = blob_at(sha, path)
     return unless blob
@@ -1060,12 +1080,6 @@ class Repository
     blob.data
   end
 
-  def refs_directory_exists?
-    circuit_breaker.perform do
-      File.exist?(File.join(path_to_repo, 'refs'))
-    end
-  end
-
   def cache
     # TODO: should we use UUIDs here? We could move repositories without clearing this cache
     @cache ||= RepositoryCache.new(full_path, @project.id)
@@ -1096,17 +1110,17 @@ class Repository
 
   def last_commit_for_path_by_gitaly(sha, path)
     c = raw_repository.gitaly_commit_client.last_commit_for_path(sha, path)
-    commit(c)
+    commit_by(oid: c)
   end
 
   def last_commit_for_path_by_rugged(sha, path)
     sha = last_commit_id_for_path_by_shelling_out(sha, path)
-    commit(sha)
+    commit_by(oid: sha)
   end
 
   def last_commit_id_for_path_by_shelling_out(sha, path)
     args = %W(rev-list --max-count=1 #{sha} -- #{path})
-    run_git(args).first.strip
+    raw_repository.run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip
   end
 
   def repository_storage_path
@@ -1114,11 +1128,7 @@ class Repository
   end
 
   def initialize_raw_repository
-    Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, false))
-  end
-
-  def circuit_breaker
-    @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(project.repository_storage)
+    Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, is_wiki))
   end
 
   def find_commits_by_message_by_shelling_out(query, ref, path, limit, offset)
diff --git a/app/models/sent_notification.rb b/app/models/sent_notification.rb
index 298569cb7a6..6e311806be1 100644
--- a/app/models/sent_notification.rb
+++ b/app/models/sent_notification.rb
@@ -53,13 +53,17 @@ class SentNotification < ActiveRecord::Base
   end
 
   def unsubscribable?
-    !for_commit?
+    !(for_commit? || for_snippet?)
   end
 
   def for_commit?
     noteable_type == "Commit"
   end
 
+  def for_snippet?
+    noteable_type.end_with?('Snippet')
+  end
+
   def noteable
     if for_commit?
       project.commit(commit_id) rescue nil
diff --git a/app/models/service.rb b/app/models/service.rb
index 6b64079215f..fdd2605e3e3 100644
--- a/app/models/service.rb
+++ b/app/models/service.rb
@@ -238,6 +238,7 @@ class Service < ActiveRecord::Base
       kubernetes
       mattermost_slash_commands
       mattermost
+      packagist
       pipelines_email
       pivotaltracker
       prometheus
diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb
index fae1b64961a..f025f40994e 100644
--- a/app/models/storage/hashed_project.rb
+++ b/app/models/storage/hashed_project.rb
@@ -4,6 +4,7 @@ module Storage
     delegate :gitlab_shell, :repository_storage_path, to: :project
 
     ROOT_PATH_PREFIX = '@hashed'.freeze
+    STORAGE_VERSION = 1
 
     def initialize(project)
       @project = project
diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb
index 0b33e45473b..1f9f8d7286b 100644
--- a/app/models/system_note_metadata.rb
+++ b/app/models/system_note_metadata.rb
@@ -2,7 +2,7 @@ class SystemNoteMetadata < ActiveRecord::Base
   ICON_TYPES = %w[
     commit description merge confidential visible label assignee cross_reference
     title time_tracking branch milestone discussion task moved
-    opened closed merged duplicate
+    opened closed merged duplicate locked unlocked
     outdated
   ].freeze
 
diff --git a/app/models/user.rb b/app/models/user.rb
index 09c9b3250eb..bcda4564595 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -21,8 +21,8 @@ class User < ActiveRecord::Base
 
   ignore_column :external_email
   ignore_column :email_provider
+  ignore_column :authentication_token
 
-  add_authentication_token_field :authentication_token
   add_authentication_token_field :incoming_email_token
   add_authentication_token_field :rss_token
 
@@ -60,7 +60,7 @@ class User < ActiveRecord::Base
     lease = Gitlab::ExclusiveLease.new("user_update_tracked_fields:#{id}", timeout: 1.hour.to_i)
     return unless lease.try_obtain
 
-    Users::UpdateService.new(self).execute(validate: false)
+    Users::UpdateService.new(self, user: self).execute(validate: false)
   end
 
   attr_accessor :force_random_password
@@ -130,6 +130,8 @@ class User < ActiveRecord::Base
   has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue
   has_many :assigned_merge_requests,  dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent
 
+  has_many :custom_attributes, class_name: 'UserCustomAttribute'
+
   #
   # Validations
   #
@@ -161,15 +163,17 @@ class User < ActiveRecord::Base
   before_validation :sanitize_attrs
   before_validation :set_notification_email, if: :email_changed?
   before_validation :set_public_email, if: :public_email_changed?
-
-  after_update :update_emails_with_primary_email, if: :email_changed?
-  before_save :ensure_authentication_token, :ensure_incoming_email_token
+  before_save :ensure_incoming_email_token
   before_save :ensure_user_rights_and_limits, if: :external_changed?
   before_save :skip_reconfirmation!, if: ->(user) { user.email_changed? && user.read_only_attribute?(:email) }
+  before_save :check_for_verified_email, if: ->(user) { user.email_changed? && !user.new_record? }
   after_save :ensure_namespace_correct
+  after_update :username_changed_hook, if: :username_changed?
+  after_destroy :post_destroy_hook
+  after_commit :update_emails_with_primary_email, on: :update, if: -> { previous_changes.key?('email') }
   after_commit :update_invalid_gpg_signatures, on: :update, if: -> { previous_changes.key?('email') }
+
   after_initialize :set_projects_limit
-  after_destroy :post_destroy_hook
 
   # User's Layout preference
   enum layout: [:fixed, :fluid]
@@ -179,15 +183,8 @@ class User < ActiveRecord::Base
   enum dashboard: [:projects, :stars, :project_activity, :starred_project_activity, :groups, :todos]
 
   # User's Project preference
-  #
-  # Note: When adding an option, it MUST go on the end of the hash with a
-  # number higher than the current max. We cannot move options and/or change
-  # their numbers.
-  #
-  # We skip 0 because this was used by an option that has since been removed.
-  enum project_view: { activity: 1, files: 2 }
-
-  alias_attribute :private_token, :authentication_token
+  # Note: When adding an option, it MUST go on the end of the array.
+  enum project_view: [:readme, :activity, :files]
 
   delegate :path, to: :namespace, allow_nil: true, prefix: true
 
@@ -456,6 +453,14 @@ class User < ActiveRecord::Base
     reset_password_sent_at.present? && reset_password_sent_at >= 1.minute.ago
   end
 
+  def remember_me!
+    super if ::Gitlab::Database.read_write?
+  end
+
+  def forget_me!
+    super if ::Gitlab::Database.read_write?
+  end
+
   def disable_two_factor!
     transaction do
       update_attributes(
@@ -523,12 +528,24 @@ class User < ActiveRecord::Base
     errors.add(:public_email, "is not an email you own") unless all_emails.include?(public_email)
   end
 
+  # see if the new email is already a verified secondary email
+  def check_for_verified_email
+    skip_reconfirmation! if emails.confirmed.where(email: self.email).any?
+  end
+
+  # Note: the use of the Emails services will cause `saves` on the user object, running
+  # through the callbacks again and can have side effects, such as the `previous_changes`
+  # hash and `_was` variables getting munged.
+  # By using an `after_commit` instead of `after_update`, we avoid the recursive callback
+  # scenario, though it then requires us to use the `previous_changes` hash
   def update_emails_with_primary_email
+    previous_email = previous_changes[:email][0]  # grab this before the DestroyService is called
     primary_email_record = emails.find_by(email: email)
-    if primary_email_record
-      Emails::DestroyService.new(self, email: email).execute
-      Emails::CreateService.new(self, email: email_was).execute
-    end
+    Emails::DestroyService.new(self, user: self).execute(primary_email_record) if primary_email_record
+
+    # the original primary email was confirmed, and we want that to carry over.  We don't
+    # have access to the original confirmation values at this point, so just set confirmed_at
+    Emails::CreateService.new(self, user: self, email: previous_email).execute(confirmed_at: confirmed_at)
   end
 
   def update_invalid_gpg_signatures
@@ -639,6 +656,10 @@ class User < ActiveRecord::Base
     Ability.allowed?(self, action, subject)
   end
 
+  def confirm_deletion_with_password?
+    !password_automatically_set? && allow_password_authentication?
+  end
+
   def first_name
     name.split.first unless name.blank?
   end
@@ -678,19 +699,15 @@ class User < ActiveRecord::Base
   end
 
   def fork_of(project)
-    links = ForkedProjectLink.where(
-      forked_from_project_id: project,
-      forked_to_project_id: personal_projects.unscope(:order)
-    )
-    if links.any?
-      links.first.forked_to_project
-    else
-      nil
-    end
+    namespace.find_fork_of(project)
   end
 
   def ldap_user?
-    identities.exists?(["provider LIKE ? AND extern_uid IS NOT NULL", "ldap%"])
+    if identities.loaded?
+      identities.find { |identity| identity.provider.start_with?('ldap') && !identity.extern_uid.nil? }
+    else
+      identities.exists?(["provider LIKE ? AND extern_uid IS NOT NULL", "ldap%"])
+    end
   end
 
   def ldap_identity
@@ -810,6 +827,10 @@ class User < ActiveRecord::Base
     avatar_path(args) || GravatarService.new.execute(email, size, scale, username: username)
   end
 
+  def primary_email_verified?
+    confirmed? && !temp_oauth_email?
+  end
+
   def all_emails
     all_emails = []
     all_emails << email unless temp_oauth_email?
@@ -817,6 +838,18 @@ class User < ActiveRecord::Base
     all_emails
   end
 
+  def verified_emails
+    verified_emails = []
+    verified_emails << email if primary_email_verified?
+    verified_emails.concat(emails.confirmed.pluck(:email))
+    verified_emails
+  end
+
+  def verified_email?(check_email)
+    downcased = check_email.downcase
+    email == downcased ? primary_email_verified? : emails.confirmed.where(email: downcased).exists?
+  end
+
   def hook_attrs
     {
       name: name,
@@ -839,6 +872,10 @@ class User < ActiveRecord::Base
     end
   end
 
+  def username_changed_hook
+    system_hook_service.execute_hooks_for(self, :rename)
+  end
+
   def post_destroy_hook
     log_info("User \"#{name}\" (#{email})  was removed")
     system_hook_service.execute_hooks_for(self, :destroy)
@@ -1000,7 +1037,7 @@ class User < ActiveRecord::Base
     if attempts_exceeded?
       lock_access! unless access_locked?
     else
-      Users::UpdateService.new(self).execute(validate: false)
+      Users::UpdateService.new(self, user: self).execute(validate: false)
     end
   end
 
@@ -1041,10 +1078,6 @@ class User < ActiveRecord::Base
     ensure_rss_token!
   end
 
-  def verified_email?(email)
-    self.email == email
-  end
-
   def sync_attribute?(attribute)
     return true if ldap_user? && attribute == :email
 
@@ -1061,6 +1094,12 @@ class User < ActiveRecord::Base
     user_synced_attributes_metadata&.read_only?(attribute)
   end
 
+  # override, from Devise
+  def lock_access!
+    Gitlab::AppLogger.info("Account Locked: username=#{username}")
+    super
+  end
+
   protected
 
   # override, from Devise::Validatable
@@ -1186,7 +1225,7 @@ class User < ActiveRecord::Base
       &creation_block
     )
 
-    Users::UpdateService.new(user).execute(validate: false)
+    Users::UpdateService.new(user, user: user).execute(validate: false)
     user
   ensure
     Gitlab::ExclusiveLease.cancel(lease_key, uuid)
diff --git a/app/models/user_custom_attribute.rb b/app/models/user_custom_attribute.rb
new file mode 100644
index 00000000000..eff25b31f9b
--- /dev/null
+++ b/app/models/user_custom_attribute.rb
@@ -0,0 +1,6 @@
+class UserCustomAttribute < ActiveRecord::Base
+  belongs_to :user
+
+  validates :user_id, :key, :value, presence: true
+  validates :key, uniqueness: { scope: [:user_id] }
+end
diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb
index f2315bb3dbb..5f710961f95 100644
--- a/app/models/wiki_page.rb
+++ b/app/models/wiki_page.rb
@@ -50,7 +50,7 @@ class WikiPage
   # The Gitlab ProjectWiki instance.
   attr_reader :wiki
 
-  # The raw Gollum::Page instance.
+  # The raw Gitlab::Git::WikiPage instance.
   attr_reader :page
 
   # The attributes Hash used for storing and validating
@@ -75,7 +75,7 @@ class WikiPage
     if @attributes[:slug].present?
       @attributes[:slug]
     else
-      wiki.wiki.preview_page(title, '', format).url_path
+      wiki.wiki.preview_slug(title, format)
     end
   end
 
@@ -131,7 +131,7 @@ class WikiPage
   def versions
     return [] unless persisted?
 
-    @page.versions
+    wiki.wiki.page_versions(@page.path)
   end
 
   def commit
@@ -264,8 +264,8 @@ class WikiPage
     end
 
     page_title, page_dir = wiki.page_title_and_dir(page_details)
-    gollum_wiki = wiki.wiki
-    @page = gollum_wiki.paged(page_title, page_dir)
+    gitlab_git_wiki = wiki.wiki
+    @page = gitlab_git_wiki.page(title: page_title, dir: page_dir)
 
     set_attributes
     @persisted = errors.blank?