Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (671 commits)
  Make rubocop happy
  Use guard clause
  Improve language
  Prettify
  Use temp branch
  Pass info about who started the job and which job triggered it
  Docs: add indexes for monitoring and performance monitoring
  clearer-documentation-on-inline-diffs
  Add docs for commit diff discussion in merge requests
  sorting for tags api
  Clear BatchLoader after each spec to prevent holding onto records longer than necessary
  Include project in BatchLoader key to prevent returning blobs for the wrong project
  moved lfs_blob_ids method into ExtractsPath module
  Converted JS modules into exported modules
  spec fixes
  Bump gitlab-shell version to 5.10.3
  Clear caches before updating MR diffs
  Use new Ruby version 2.4 in GitLab QA images
  moved lfs blob fetch from extractspath file
  Update GitLab QA dependencies
  ...

68 files changed, 1864 insertions, 422 deletions

diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb
new file mode 100644
index 00000000000..81e95e5832d
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb
@@ -0,0 +1,259 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This class processes a batch of rows in `untracked_files_for_uploads` by
+    # adding each file to the `uploads` table if it does not exist.
+    class PopulateUntrackedUploads # rubocop:disable Metrics/ClassLength
+      # This class is responsible for producing the attributes necessary to
+      # track an uploaded file in the `uploads` table.
+      class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength
+        self.table_name = 'untracked_files_for_uploads'
+
+        # Ends with /:random_hex/:filename
+        FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z}
+        FULL_PATH_CAPTURE = %r{\A(.+)#{FILE_UPLOADER_PATH}}
+
+        # These regex patterns are tested against a relative path, relative to
+        # the upload directory.
+        # For convenience, if there exists a capture group in the pattern, then
+        # it indicates the model_id.
+        PATH_PATTERNS = [
+          {
+            pattern: %r{\A-/system/appearance/logo/(\d+)/},
+            uploader: 'AttachmentUploader',
+            model_type: 'Appearance'
+          },
+          {
+            pattern: %r{\A-/system/appearance/header_logo/(\d+)/},
+            uploader: 'AttachmentUploader',
+            model_type: 'Appearance'
+          },
+          {
+            pattern: %r{\A-/system/note/attachment/(\d+)/},
+            uploader: 'AttachmentUploader',
+            model_type: 'Note'
+          },
+          {
+            pattern: %r{\A-/system/user/avatar/(\d+)/},
+            uploader: 'AvatarUploader',
+            model_type: 'User'
+          },
+          {
+            pattern: %r{\A-/system/group/avatar/(\d+)/},
+            uploader: 'AvatarUploader',
+            model_type: 'Namespace'
+          },
+          {
+            pattern: %r{\A-/system/project/avatar/(\d+)/},
+            uploader: 'AvatarUploader',
+            model_type: 'Project'
+          },
+          {
+            pattern: FILE_UPLOADER_PATH,
+            uploader: 'FileUploader',
+            model_type: 'Project'
+          }
+        ].freeze
+
+        def to_h
+          @upload_hash ||= {
+            path: upload_path,
+            uploader: uploader,
+            model_type: model_type,
+            model_id: model_id,
+            size: file_size,
+            checksum: checksum
+          }
+        end
+
+        def upload_path
+          # UntrackedFile#path is absolute, but Upload#path depends on uploader
+          @upload_path ||=
+            if uploader == 'FileUploader'
+              # Path relative to project directory in uploads
+              matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH)
+              matchd[0].sub(%r{\A/}, '') # remove leading slash
+            else
+              path
+            end
+        end
+
+        def uploader
+          matching_pattern_map[:uploader]
+        end
+
+        def model_type
+          matching_pattern_map[:model_type]
+        end
+
+        def model_id
+          return @model_id if defined?(@model_id)
+
+          pattern = matching_pattern_map[:pattern]
+          matchd = path_relative_to_upload_dir.match(pattern)
+
+          # If something is captured (matchd[1] is not nil), it is a model_id
+          # Only the FileUploader pattern will not match an ID
+          @model_id = matchd[1] ? matchd[1].to_i : file_uploader_model_id
+        end
+
+        def file_size
+          File.size(absolute_path)
+        end
+
+        def checksum
+          Digest::SHA256.file(absolute_path).hexdigest
+        end
+
+        private
+
+        def matching_pattern_map
+          @matching_pattern_map ||= PATH_PATTERNS.find do |path_pattern_map|
+            path_relative_to_upload_dir.match(path_pattern_map[:pattern])
+          end
+
+          unless @matching_pattern_map
+            raise "Unknown upload path pattern \"#{path}\""
+          end
+
+          @matching_pattern_map
+        end
+
+        def file_uploader_model_id
+          matchd = path_relative_to_upload_dir.match(FULL_PATH_CAPTURE)
+          not_found_msg = <<~MSG
+            Could not capture project full_path from a FileUploader path:
+              "#{path_relative_to_upload_dir}"
+          MSG
+          raise not_found_msg unless matchd
+
+          full_path = matchd[1]
+          project = Project.find_by_full_path(full_path)
+          return nil unless project
+
+          project.id
+        end
+
+        # Not including a leading slash
+        def path_relative_to_upload_dir
+          upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR # rubocop:disable Metrics/LineLength
+          base = %r{\A#{Regexp.escape(upload_dir)}/}
+          @path_relative_to_upload_dir ||= path.sub(base, '')
+        end
+
+        def absolute_path
+          File.join(CarrierWave.root, path)
+        end
+      end
+
+      # This class is used to query the `uploads` table.
+      class Upload < ActiveRecord::Base
+        self.table_name = 'uploads'
+      end
+
+      def perform(start_id, end_id)
+        return unless migrate?
+
+        files = UntrackedFile.where(id: start_id..end_id)
+        processed_files = insert_uploads_if_needed(files)
+        processed_files.delete_all
+
+        drop_temp_table_if_finished
+      end
+
+      private
+
+      def migrate?
+        UntrackedFile.table_exists? && Upload.table_exists?
+      end
+
+      def insert_uploads_if_needed(files)
+        filtered_files, error_files = filter_error_files(files)
+        filtered_files = filter_existing_uploads(filtered_files)
+        filtered_files = filter_deleted_models(filtered_files)
+        insert(filtered_files)
+
+        processed_files = files.where.not(id: error_files.map(&:id))
+        processed_files
+      end
+
+      def filter_error_files(files)
+        files.partition do |file|
+          begin
+            file.to_h
+            true
+          rescue => e
+            msg = <<~MSG
+              Error parsing path "#{file.path}":
+                #{e.message}
+                #{e.backtrace.join("\n  ")}
+            MSG
+            Rails.logger.error(msg)
+            false
+          end
+        end
+      end
+
+      def filter_existing_uploads(files)
+        paths = files.map(&:upload_path)
+        existing_paths = Upload.where(path: paths).pluck(:path).to_set
+
+        files.reject do |file|
+          existing_paths.include?(file.upload_path)
+        end
+      end
+
+      # There are files on disk that are not in the uploads table because their
+      # model was deleted, and we don't delete the files on disk.
+      def filter_deleted_models(files)
+        ids = deleted_model_ids(files)
+
+        files.reject do |file|
+          ids[file.model_type].include?(file.model_id)
+        end
+      end
+
+      def deleted_model_ids(files)
+        ids = {
+          'Appearance' => [],
+          'Namespace' => [],
+          'Note' => [],
+          'Project' => [],
+          'User' => []
+        }
+
+        # group model IDs by model type
+        files.each do |file|
+          ids[file.model_type] << file.model_id
+        end
+
+        ids.each do |model_type, model_ids|
+          model_class = Object.const_get(model_type)
+          found_ids = model_class.where(id: model_ids.uniq).pluck(:id)
+          deleted_ids = ids[model_type] - found_ids
+          ids[model_type] = deleted_ids
+        end
+
+        ids
+      end
+
+      def insert(files)
+        rows = files.map do |file|
+          file.to_h.merge(created_at: 'NOW()')
+        end
+
+        Gitlab::Database.bulk_insert('uploads',
+                                     rows,
+                                     disable_quote: :created_at)
+      end
+
+      def drop_temp_table_if_finished
+        if UntrackedFile.all.empty?
+          UntrackedFile.connection.drop_table(:untracked_files_for_uploads,
+                                              if_exists: true)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb
new file mode 100644
index 00000000000..476c46341ae
--- /dev/null
+++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb
@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This class finds all non-hashed uploaded file paths and saves them to a
+    # `untracked_files_for_uploads` table.
+    class PrepareUntrackedUploads # rubocop:disable Metrics/ClassLength
+      # For bulk_queue_background_migration_jobs_by_range
+      include Database::MigrationHelpers
+
+      FIND_BATCH_SIZE = 500
+      RELATIVE_UPLOAD_DIR = "uploads".freeze
+      ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze
+      FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
+      START_WITH_CARRIERWAVE_ROOT_REGEX = %r{\A#{CarrierWave.root}/}
+      EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze
+      EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze
+
+      # This class is used to iterate over batches of
+      # `untracked_files_for_uploads` rows.
+      class UntrackedFile < ActiveRecord::Base
+        include EachBatch
+
+        self.table_name = 'untracked_files_for_uploads'
+      end
+
+      def perform
+        ensure_temporary_tracking_table_exists
+
+        # Since Postgres < 9.5 does not have ON CONFLICT DO NOTHING, and since
+        # doing inserts-if-not-exists without ON CONFLICT DO NOTHING would be
+        # slow, start with an empty table for Postgres < 9.5.
+        # That way we can do bulk inserts at ~30x the speed of individual
+        # inserts (~20 minutes worth of inserts at GitLab.com scale instead of
+        # ~10 hours).
+        # In all other cases, installations will get both bulk inserts and the
+        # ability for these jobs to retry without having to clear and reinsert.
+        clear_untracked_file_paths unless can_bulk_insert_and_ignore_duplicates?
+
+        store_untracked_file_paths
+
+        schedule_populate_untracked_uploads_jobs
+      end
+
+      private
+
+      def ensure_temporary_tracking_table_exists
+        table_name = :untracked_files_for_uploads
+        unless UntrackedFile.connection.table_exists?(table_name)
+          UntrackedFile.connection.create_table table_name do |t|
+            t.string :path, limit: 600, null: false
+            t.index :path, unique: true
+          end
+        end
+      end
+
+      def clear_untracked_file_paths
+        UntrackedFile.delete_all
+      end
+
+      def store_untracked_file_paths
+        return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR)
+
+        each_file_batch(ABSOLUTE_UPLOAD_DIR, FIND_BATCH_SIZE) do |file_paths|
+          insert_file_paths(file_paths)
+        end
+      end
+
+      def each_file_batch(search_dir, batch_size, &block)
+        cmd = build_find_command(search_dir)
+
+        Open3.popen2(*cmd) do |stdin, stdout, status_thread|
+          yield_paths_in_batches(stdout, batch_size, &block)
+
+          raise "Find command failed" unless status_thread.value.success?
+        end
+      end
+
+      def yield_paths_in_batches(stdout, batch_size, &block)
+        paths = []
+
+        stdout.each_line("\0") do |line|
+          paths << line.chomp("\0").sub(START_WITH_CARRIERWAVE_ROOT_REGEX, '')
+
+          if paths.size >= batch_size
+            yield(paths)
+            paths = []
+          end
+        end
+
+        yield(paths)
+      end
+
+      def build_find_command(search_dir)
+        cmd = %W[find -L #{search_dir}
+                 -type f
+                 ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune )
+                 ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune )
+                 -print0]
+
+        ionice = which_ionice
+        cmd = %W[#{ionice} -c Idle] + cmd if ionice
+
+        log_msg = "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\""
+        Rails.logger.info log_msg
+
+        cmd
+      end
+
+      def which_ionice
+        Gitlab::Utils.which('ionice')
+      rescue StandardError
+        # In this case, returning false is relatively safe,
+        # even though it isn't very nice
+        false
+      end
+
+      def insert_file_paths(file_paths)
+        sql = insert_sql(file_paths)
+
+        ActiveRecord::Base.connection.execute(sql)
+      end
+
+      def insert_sql(file_paths)
+        if postgresql_pre_9_5?
+          "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};"
+        elsif postgresql?
+          "INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\
+            " ON CONFLICT DO NOTHING;"
+        else # MySQL
+          "INSERT IGNORE INTO"\
+            " #{table_columns_and_values_for_insert(file_paths)};"
+        end
+      end
+
+      def table_columns_and_values_for_insert(file_paths)
+        values = file_paths.map do |file_path|
+          ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend, Metrics/LineLength
+        end.join(', ')
+
+        "#{UntrackedFile.table_name} (path) VALUES #{values}"
+      end
+
+      def postgresql?
+        @postgresql ||= Gitlab::Database.postgresql?
+      end
+
+      def can_bulk_insert_and_ignore_duplicates?
+        !postgresql_pre_9_5?
+      end
+
+      def postgresql_pre_9_5?
+        @postgresql_pre_9_5 ||= postgresql? &&
+          Gitlab::Database.version.to_f < 9.5
+      end
+
+      def schedule_populate_untracked_uploads_jobs
+        bulk_queue_background_migration_jobs_by_range(
+          UntrackedFile, FOLLOW_UP_MIGRATION)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb
index 196de667805..298409d8b5a 100644
--- a/lib/gitlab/bare_repository_import/importer.rb
+++ b/lib/gitlab/bare_repository_import/importer.rb
@@ -55,6 +55,7 @@ module Gitlab
                                               name: project_name,
                                               path: project_name,
                                               skip_disk_validation: true,
+                                              import_type: 'gitlab_project',
                                               namespace_id: group&.id).execute
 
         if project.persisted? && mv_repo(project)
diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb
index 8574ac6eb30..fa7891c8dcc 100644
--- a/lib/gitlab/bare_repository_import/repository.rb
+++ b/lib/gitlab/bare_repository_import/repository.rb
@@ -7,6 +7,8 @@ module Gitlab
         @root_path = root_path
         @repo_path = repo_path
 
+        @root_path << '/' unless root_path.ends_with?('/')
+
         # Split path into 'all/the/namespaces' and 'project_name'
         @group_path, _, @project_name = repo_relative_path.rpartition('/')
       end
diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb
new file mode 100644
index 00000000000..3a1c0a3455e
--- /dev/null
+++ b/lib/gitlab/checks/project_moved.rb
@@ -0,0 +1,65 @@
+module Gitlab
+  module Checks
+    class ProjectMoved
+      REDIRECT_NAMESPACE = "redirect_namespace".freeze
+
+      def initialize(project, user, redirected_path, protocol)
+        @project = project
+        @user = user
+        @redirected_path = redirected_path
+        @protocol = protocol
+      end
+
+      def self.fetch_redirect_message(user_id, project_id)
+        redirect_key = redirect_message_key(user_id, project_id)
+
+        Gitlab::Redis::SharedState.with do |redis|
+          message = redis.get(redirect_key)
+          redis.del(redirect_key)
+          message
+        end
+      end
+
+      def add_redirect_message
+        Gitlab::Redis::SharedState.with do |redis|
+          key = self.class.redirect_message_key(user.id, project.id)
+          redis.setex(key, 5.minutes, redirect_message)
+        end
+      end
+
+      def redirect_message(rejected: false)
+        <<~MESSAGE.strip_heredoc
+        Project '#{redirected_path}' was moved to '#{project.full_path}'.
+
+        Please update your Git remote:
+
+          #{remote_url_message(rejected)}
+        MESSAGE
+      end
+
+      def permanent_redirect?
+        RedirectRoute.permanent.exists?(path: redirected_path)
+      end
+
+      private
+
+      attr_reader :project, :redirected_path, :protocol, :user
+
+      def self.redirect_message_key(user_id, project_id)
+        "#{REDIRECT_NAMESPACE}:#{user_id}:#{project_id}"
+      end
+
+      def remote_url_message(rejected)
+        if rejected
+          "git remote set-url origin #{url} and try again."
+        else
+          "git remote set-url origin #{url}"
+        end
+      end
+
+      def url
+        protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb
index 8d82e1b288d..efed19da21c 100644
--- a/lib/gitlab/ci/pipeline/chain/base.rb
+++ b/lib/gitlab/ci/pipeline/chain/base.rb
@@ -3,14 +3,13 @@ module Gitlab
     module Pipeline
       module Chain
         class Base
-          attr_reader :pipeline, :project, :current_user
+          attr_reader :pipeline, :command
+
+          delegate :project, :current_user, to: :command
 
           def initialize(pipeline, command)
             @pipeline = pipeline
             @command = command
-
-            @project = command.project
-            @current_user = command.current_user
           end
 
           def perform!
diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb
new file mode 100644
index 00000000000..70732d26bbd
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/build.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class Build < Chain::Base
+          def perform!
+            @pipeline.assign_attributes(
+              source: @command.source,
+              project: @command.project,
+              ref: @command.ref,
+              sha: @command.sha,
+              before_sha: @command.before_sha,
+              tag: @command.tag_exists?,
+              trigger_requests: Array(@command.trigger_request),
+              user: @command.current_user,
+              pipeline_schedule: @command.schedule,
+              protected: @command.protected_ref?
+            )
+
+            @pipeline.set_config_source
+          end
+
+          def break?
+            false
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
new file mode 100644
index 00000000000..7b19b10e05b
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -0,0 +1,61 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        Command = Struct.new(
+          :source, :project, :current_user,
+          :origin_ref, :checkout_sha, :after_sha, :before_sha,
+          :trigger_request, :schedule,
+          :ignore_skip_ci, :save_incompleted,
+          :seeds_block
+        ) do
+          include Gitlab::Utils::StrongMemoize
+
+          def initialize(**params)
+            params.each do |key, value|
+              self[key] = value
+            end
+          end
+
+          def branch_exists?
+            strong_memoize(:is_branch) do
+              project.repository.branch_exists?(ref)
+            end
+          end
+
+          def tag_exists?
+            strong_memoize(:is_tag) do
+              project.repository.tag_exists?(ref)
+            end
+          end
+
+          def ref
+            strong_memoize(:ref) do
+              Gitlab::Git.ref_name(origin_ref)
+            end
+          end
+
+          def sha
+            strong_memoize(:sha) do
+              project.commit(origin_sha || origin_ref).try(:id)
+            end
+          end
+
+          def origin_sha
+            checkout_sha || after_sha
+          end
+
+          def before_sha
+            self[:before_sha] || checkout_sha || Gitlab::Git::BLANK_SHA
+          end
+
+          def protected_ref?
+            strong_memoize(:protected_ref) do
+              project.protected_for?(ref)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb
index d5e17a123df..d19a2519803 100644
--- a/lib/gitlab/ci/pipeline/chain/create.rb
+++ b/lib/gitlab/ci/pipeline/chain/create.rb
@@ -17,11 +17,27 @@ module Gitlab
             end
           rescue ActiveRecord::RecordInvalid => e
             error("Failed to persist the pipeline: #{e}")
+          ensure
+            if pipeline.builds.where(stage_id: nil).any?
+              invalid_builds_counter.increment(node: hostname)
+            end
           end
 
           def break?
             !pipeline.persisted?
           end
+
+          private
+
+          def invalid_builds_counter
+            @counter ||= Gitlab::Metrics
+              .counter(:gitlab_ci_invalid_builds_total,
+                       'Invalid builds without stage assigned counter')
+          end
+
+          def hostname
+            @hostname ||= Socket.gethostname
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb
index 7ab7a64c7e3..bf1380a1da9 100644
--- a/lib/gitlab/ci/pipeline/chain/helpers.rb
+++ b/lib/gitlab/ci/pipeline/chain/helpers.rb
@@ -3,20 +3,6 @@ module Gitlab
     module Pipeline
       module Chain
         module Helpers
-          include Gitlab::Utils::StrongMemoize
-
-          def branch_exists?
-            strong_memoize(:is_branch) do
-              project.repository.branch_exists?(pipeline.ref)
-            end
-          end
-
-          def tag_exists?
-            strong_memoize(:is_tag) do
-              project.repository.tag_exists?(pipeline.ref)
-            end
-          end
-
           def error(message)
             pipeline.errors.add(:base, message)
           end
diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb
index 015f2988327..e24630656d3 100644
--- a/lib/gitlab/ci/pipeline/chain/sequence.rb
+++ b/lib/gitlab/ci/pipeline/chain/sequence.rb
@@ -5,20 +5,19 @@ module Gitlab
         class Sequence
           def initialize(pipeline, command, sequence)
             @pipeline = pipeline
+            @command = command
+            @sequence = sequence
             @completed = []
-
-            @sequence = sequence.map do |chain|
-              chain.new(pipeline, command)
-            end
           end
 
           def build!
-            @sequence.each do |step|
-              step.perform!
+            @sequence.each do |chain|
+              step = chain.new(@pipeline, @command)
 
+              step.perform!
               break if step.break?
 
-              @completed << step
+              @completed.push(step)
             end
 
             @pipeline.tap do
diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
index 4913a604079..13c6fedd831 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
@@ -14,7 +14,7 @@ module Gitlab
 
               unless allowed_to_trigger_pipeline?
                 if can?(current_user, :create_pipeline, project)
-                  return error("Insufficient permissions for protected ref '#{pipeline.ref}'")
+                  return error("Insufficient permissions for protected ref '#{command.ref}'")
                 else
                   return error('Insufficient permissions to create a new pipeline')
                 end
@@ -29,7 +29,7 @@ module Gitlab
               if current_user
                 allowed_to_create?
               else # legacy triggers don't have a corresponding user
-                !project.protected_for?(@pipeline.ref)
+                !@command.protected_ref?
               end
             end
 
@@ -38,10 +38,10 @@ module Gitlab
 
               access = Gitlab::UserAccess.new(current_user, project: project)
 
-              if branch_exists?
-                access.can_update_branch?(@pipeline.ref)
-              elsif tag_exists?
-                access.can_create_tag?(@pipeline.ref)
+              if @command.branch_exists?
+                access.can_update_branch?(@command.ref)
+              elsif @command.tag_exists?
+                access.can_create_tag?(@command.ref)
               else
                 true # Allow it for now and we'll reject when we check ref existence
               end
diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb
index 70a4cfdbdea..9699c24e5b6 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/repository.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb
@@ -7,14 +7,11 @@ module Gitlab
             include Chain::Helpers
 
             def perform!
-              unless branch_exists? || tag_exists?
+              unless @command.branch_exists? || @command.tag_exists?
                 return error('Reference not found')
               end
 
-              ## TODO, we check commit in the service, that is why
-              # there is no repository access here.
-              #
-              unless pipeline.sha
+              unless @command.sha
                 return error('Commit not found')
               end
             end
diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb
index fb28e80ff73..b9099ce256a 100644
--- a/lib/gitlab/conflict/file_collection.rb
+++ b/lib/gitlab/conflict/file_collection.rb
@@ -19,6 +19,8 @@ module Gitlab
           commit_message: commit_message || default_commit_message
         }
         resolver.resolve_conflicts(user, files, args)
+      ensure
+        @merge_request.clear_memoized_shas
       end
 
       def files
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index cd7b4c043da..e51794fef99 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -50,6 +50,10 @@ module Gitlab
       postgresql? && version.to_f >= 9.3
     end
 
+    def self.replication_slots_supported?
+      postgresql? && version.to_f >= 9.4
+    end
+
     def self.nulls_last_order(field, direction = 'ASC')
       order = "#{field} #{direction}"
 
@@ -116,15 +120,21 @@ module Gitlab
     #        values.
     # return_ids - When set to true the return value will be an Array of IDs of
     #              the inserted rows, this only works on PostgreSQL.
-    def self.bulk_insert(table, rows, return_ids: false)
+    # disable_quote - A key or an Array of keys to exclude from quoting (You
+    #                 become responsible for protection from SQL injection for
+    #                 these keys!)
+    def self.bulk_insert(table, rows, return_ids: false, disable_quote: [])
       return if rows.empty?
 
       keys = rows.first.keys
       columns = keys.map { |key| connection.quote_column_name(key) }
       return_ids = false if mysql?
 
+      disable_quote = Array(disable_quote).to_set
       tuples = rows.map do |row|
-        row.values_at(*keys).map { |value| connection.quote(value) }
+        keys.map do |k|
+          disable_quote.include?(k) ? row[k] : connection.quote(row[k])
+        end
       end
 
       sql = <<-EOF
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index c276c3566b4..3f65bc912de 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -703,14 +703,14 @@ into similar problems in the future (e.g. when new tables are created).
             # We push multiple jobs at a time to reduce the time spent in
             # Sidekiq/Redis operations. We're using this buffer based approach so we
             # don't need to run additional queries for every range.
-            BackgroundMigrationWorker.perform_bulk(jobs)
+            BackgroundMigrationWorker.bulk_perform_async(jobs)
             jobs.clear
           end
 
           jobs << [job_class_name, [start_id, end_id]]
         end
 
-        BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty?
+        BackgroundMigrationWorker.bulk_perform_async(jobs) unless jobs.empty?
       end
 
       # Queues background migration jobs for an entire table, batched by ID range.
diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb
index c98eefbce25..88e0db830f6 100644
--- a/lib/gitlab/diff/diff_refs.rb
+++ b/lib/gitlab/diff/diff_refs.rb
@@ -13,9 +13,9 @@ module Gitlab
 
       def ==(other)
         other.is_a?(self.class) &&
-          shas_equal?(base_sha, other.base_sha) &&
-          shas_equal?(start_sha, other.start_sha) &&
-          shas_equal?(head_sha, other.head_sha)
+          Git.shas_eql?(base_sha, other.base_sha) &&
+          Git.shas_eql?(start_sha, other.start_sha) &&
+          Git.shas_eql?(head_sha, other.head_sha)
       end
 
       alias_method :eql?, :==
@@ -47,22 +47,6 @@ module Gitlab
           CompareService.new(project, head_sha).execute(project, start_sha, straight: straight)
         end
       end
-
-      private
-
-      def shas_equal?(sha1, sha2)
-        return true if sha1 == sha2
-        return false if sha1.nil? || sha2.nil?
-        return false unless sha1.class == sha2.class
-
-        length = [sha1.length, sha2.length].min
-
-        # If either of the shas is below the minimum length, we cannot be sure
-        # that they actually refer to the same commit because of hash collision.
-        return false if length < Commit::MIN_SHA_LENGTH
-
-        sha1[0, length] == sha2[0, length]
-      end
     end
   end
 end
diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb
index 2d7b57120a6..54783a07919 100644
--- a/lib/gitlab/diff/inline_diff.rb
+++ b/lib/gitlab/diff/inline_diff.rb
@@ -70,7 +70,7 @@ module Gitlab
         def find_changed_line_pairs(lines)
           # Prefixes of all diff lines, indicating their types
           # For example: `" - +  -+  ---+++ --+  -++"`
-          line_prefixes = lines.each_with_object("") { |line, s| s << line[0] }.gsub(/[^ +-]/, ' ')
+          line_prefixes = lines.each_with_object("") { |line, s| s << (line[0] || ' ') }.gsub(/[^ +-]/, ' ')
 
           changed_line_pairs = []
           line_prefixes.scan(LINE_PAIRS_PATTERN) do
diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb
index 4a9d3e52fae..37face8e7d0 100644
--- a/lib/gitlab/ee_compat_check.rb
+++ b/lib/gitlab/ee_compat_check.rb
@@ -280,7 +280,7 @@ module Gitlab
         The `#{branch}` branch applies cleanly to EE/master!
 
         Much ❤️! For more information, see
-        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
+        https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html
         #{THANKS_FOR_READING_BANNER}
       }
     end
@@ -357,7 +357,7 @@ module Gitlab
         Once this is done, you can retry this failed build, and it should pass.
 
         Stay 💪 ! For more information, see
-        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
+        https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html
         #{THANKS_FOR_READING_BANNER}
       }
     end
@@ -378,7 +378,7 @@ module Gitlab
         retry this build.
 
         Stay 💪 ! For more information, see
-        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
+        https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html
         #{THANKS_FOR_READING_BANNER}
       }
     end
diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb
index b07c68d1498..e08b5be8984 100644
--- a/lib/gitlab/email/handler.rb
+++ b/lib/gitlab/email/handler.rb
@@ -1,3 +1,4 @@
+require 'gitlab/email/handler/create_merge_request_handler'
 require 'gitlab/email/handler/create_note_handler'
 require 'gitlab/email/handler/create_issue_handler'
 require 'gitlab/email/handler/unsubscribe_handler'
@@ -8,6 +9,7 @@ module Gitlab
       HANDLERS = [
         UnsubscribeHandler,
         CreateNoteHandler,
+        CreateMergeRequestHandler,
         CreateIssueHandler
       ].freeze
 
diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb
new file mode 100644
index 00000000000..e2f7c1d0257
--- /dev/null
+++ b/lib/gitlab/email/handler/create_merge_request_handler.rb
@@ -0,0 +1,69 @@
+require 'gitlab/email/handler/base_handler'
+require 'gitlab/email/handler/reply_processing'
+
+module Gitlab
+  module Email
+    module Handler
+      class CreateMergeRequestHandler < BaseHandler
+        include ReplyProcessing
+        attr_reader :project_path, :incoming_email_token
+
+        def initialize(mail, mail_key)
+          super(mail, mail_key)
+          if m = /\A([^\+]*)\+merge-request\+(.*)/.match(mail_key.to_s)
+            @project_path, @incoming_email_token = m.captures
+          end
+        end
+
+        def can_handle?
+          @project_path && @incoming_email_token
+        end
+
+        def execute
+          raise ProjectNotFound unless project
+
+          validate_permission!(:create_merge_request)
+
+          verify_record!(
+            record: create_merge_request,
+            invalid_exception: InvalidMergeRequestError,
+            record_name: 'merge_request')
+        end
+
+        def author
+          @author ||= User.find_by(incoming_email_token: incoming_email_token)
+        end
+
+        def project
+          @project ||= Project.find_by_full_path(project_path)
+        end
+
+        def metrics_params
+          super.merge(project: project&.full_path)
+        end
+
+        private
+
+        def create_merge_request
+          merge_request = MergeRequests::BuildService.new(project, author, merge_request_params).execute
+
+          if merge_request.errors.any?
+            merge_request
+          else
+            MergeRequests::CreateService.new(project, author).create(merge_request)
+          end
+        end
+
+        def merge_request_params
+          params = {
+            source_project_id: project.id,
+            source_branch: mail.subject,
+            target_project_id: project.id
+          }
+          params[:description] = message if message.present?
+          params
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index c8f4591d060..d8c594ad0e7 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -13,8 +13,10 @@ module Gitlab
     UserBlockedError = Class.new(ProcessingError)
     UserNotAuthorizedError = Class.new(ProcessingError)
     NoteableNotFoundError = Class.new(ProcessingError)
-    InvalidNoteError = Class.new(ProcessingError)
-    InvalidIssueError = Class.new(ProcessingError)
+    InvalidRecordError = Class.new(ProcessingError)
+    InvalidNoteError = Class.new(InvalidRecordError)
+    InvalidIssueError = Class.new(InvalidRecordError)
+    InvalidMergeRequestError = Class.new(InvalidRecordError)
     UnknownIncomingEmail = Class.new(ProcessingError)
 
     class Receiver
diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index 1f31cdbc96d..1f7c35cafaa 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -70,6 +70,18 @@ module Gitlab
       def diff_line_code(file_path, new_line_position, old_line_position)
         "#{Digest::SHA1.hexdigest(file_path)}_#{old_line_position}_#{new_line_position}"
       end
+
+      def shas_eql?(sha1, sha2)
+        return false if sha1.nil? || sha2.nil?
+        return false unless sha1.class == sha2.class
+
+        # If either of the shas is below the minimum length, we cannot be sure
+        # that they actually refer to the same commit because of hash collision.
+        length = [sha1.length, sha2.length].min
+        return false if length < Gitlab::Git::Commit::MIN_SHA_LENGTH
+
+        sha1[0, length] == sha2[0, length]
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index ddd52136bc4..228d97a87ab 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -49,6 +49,7 @@ module Gitlab
         # Keep in mind that this method may allocate a lot of memory. It is up
         # to the caller to limit the number of blobs and blob_size_limit.
         #
+        # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798
         def batch(repository, blob_references, blob_size_limit: nil)
           blob_size_limit ||= MAX_DATA_DISPLAY_SIZE
           blob_references.map do |sha, path|
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index d5518814483..e90b158fb34 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -6,6 +6,7 @@ module Gitlab
 
       attr_accessor :raw_commit, :head
 
+      MIN_SHA_LENGTH = 7
       SERIALIZE_KEYS = [
         :id, :message, :parent_ids,
         :authored_date, :author_name, :author_email,
@@ -213,11 +214,17 @@ module Gitlab
         end
 
         def shas_with_signatures(repository, shas)
-          shas.select do |sha|
-            begin
-              Rugged::Commit.extract_signature(repository.rugged, sha)
-            rescue Rugged::OdbError
-              false
+          GitalyClient.migrate(:filter_shas_with_signatures) do |is_enabled|
+            if is_enabled
+              Gitlab::GitalyClient::CommitService.new(repository).filter_shas_with_signatures(shas)
+            else
+              shas.select do |sha|
+                begin
+                  Rugged::Commit.extract_signature(repository.rugged, sha)
+                rescue Rugged::OdbError
+                  false
+                end
+              end
             end
           end
         end
@@ -418,6 +425,20 @@ module Gitlab
         parent_ids.size > 1
       end
 
+      def to_gitaly_commit
+        return raw_commit if raw_commit.is_a?(Gitaly::GitCommit)
+
+        message_split = raw_commit.message.split("\n", 2)
+        Gitaly::GitCommit.new(
+          id: raw_commit.oid,
+          subject: message_split[0] ? message_split[0].chomp.b : "",
+          body: raw_commit.message.b,
+          parent_ids: raw_commit.parent_ids,
+          author: gitaly_commit_author_from_rugged(raw_commit.author),
+          committer: gitaly_commit_author_from_rugged(raw_commit.committer)
+        )
+      end
+
       private
 
       def init_from_hash(hash)
@@ -463,6 +484,14 @@ module Gitlab
       def serialize_keys
         SERIALIZE_KEYS
       end
+
+      def gitaly_commit_author_from_rugged(author_or_committer)
+        Gitaly::CommitAuthor.new(
+          name: author_or_committer[:name].b,
+          email: author_or_committer[:email].b,
+          date: Google::Protobuf::Timestamp.new(seconds: author_or_committer[:time].to_i)
+        )
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb
index fc1595f1faf..b2a625e08fa 100644
--- a/lib/gitlab/git/conflict/file.rb
+++ b/lib/gitlab/git/conflict/file.rb
@@ -2,7 +2,7 @@ module Gitlab
   module Git
     module Conflict
       class File
-        attr_reader :content, :their_path, :our_path, :our_mode, :repository
+        attr_reader :content, :their_path, :our_path, :our_mode, :repository, :commit_oid
 
         def initialize(repository, commit_oid, conflict, content)
           @repository = repository
diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb
index df509c5f4ce..de8cce41b6d 100644
--- a/lib/gitlab/git/conflict/resolver.rb
+++ b/lib/gitlab/git/conflict/resolver.rb
@@ -75,7 +75,7 @@ module Gitlab
             resolved_lines = file.resolve_lines(params[:sections])
             new_file = resolved_lines.map { |line| line[:full_line] }.join("\n")
 
-            new_file << "\n" if file.our_blob.data.ends_with?("\n")
+            new_file << "\n" if file.our_blob.data.end_with?("\n")
           elsif params[:content]
             new_file = file.resolve_content(params[:content])
           end
diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb
index e36d5410431..ef5bdbaf819 100644
--- a/lib/gitlab/git/operation_service.rb
+++ b/lib/gitlab/git/operation_service.rb
@@ -83,7 +83,7 @@ module Gitlab
         Gitlab::Git.check_namespace!(start_repository)
         start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository)
 
-        start_branch_name = nil if start_repository.empty_repo?
+        start_branch_name = nil if start_repository.empty?
 
         if start_branch_name && !start_repository.branch_exists?(start_branch_name)
           raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.relative_path}"
@@ -126,7 +126,7 @@ module Gitlab
 
         oldrev = branch.target
 
-        if oldrev == repository.rugged.merge_base(newrev, branch.target)
+        if oldrev == repository.merge_base(newrev, branch.target)
           oldrev
         else
           raise Gitlab::Git::CommitError.new('Branch diverged')
diff --git a/lib/gitlab/git/remote_repository.rb b/lib/gitlab/git/remote_repository.rb
index 3685aa20669..6bd6e58feeb 100644
--- a/lib/gitlab/git/remote_repository.rb
+++ b/lib/gitlab/git/remote_repository.rb
@@ -24,10 +24,12 @@ module Gitlab
         @path = repository.path
       end
 
-      def empty_repo?
+      def empty?
         # We will override this implementation in gitaly-ruby because we cannot
         # use '@repository' there.
-        @repository.empty_repo?
+        #
+        # Caches and memoization used on the Rails side
+        !@repository.exists? || @repository.empty?
       end
 
       def commit_id(revision)
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index d399636bb28..369bb16f719 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -18,6 +18,9 @@ module Gitlab
         GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE
       ].freeze
       SEARCH_CONTEXT_LINES = 3
+      REBASE_WORKTREE_PREFIX = 'rebase'.freeze
+      SQUASH_WORKTREE_PREFIX = 'squash'.freeze
+      GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze
 
       NoRepository = Class.new(StandardError)
       InvalidBlobName = Class.new(StandardError)
@@ -73,9 +76,6 @@ module Gitlab
         @attributes = Gitlab::Git::Attributes.new(path)
       end
 
-      delegate  :empty?,
-                to: :rugged
-
       def ==(other)
         path == other.path
       end
@@ -204,6 +204,13 @@ module Gitlab
         end
       end
 
+      # Git repository can contains some hidden refs like:
+      #   /refs/notes/*
+      #   /refs/git-as-svn/*
+      #   /refs/pulls/*
+      # This refs by default not visible in project page and not cloned to client side.
+      alias_method :has_visible_content?, :has_local_branches?
+
       def has_local_branches_rugged?
         rugged.branches.each(:local).any? do |ref|
           begin
@@ -505,13 +512,20 @@ module Gitlab
 
       # Counts the amount of commits between `from` and `to`.
       def count_commits_between(from, to)
-        Commit.between(self, from, to).size
+        count_commits(ref: "#{from}..#{to}")
       end
 
       # Returns the SHA of the most recent common ancestor of +from+ and +to+
       def merge_base_commit(from, to)
-        rugged.merge_base(from, to)
+        gitaly_migrate(:merge_base) do |is_enabled|
+          if is_enabled
+            gitaly_repository_client.find_merge_base(from, to)
+          else
+            rugged.merge_base(from, to)
+          end
+        end
       end
+      alias_method :merge_base, :merge_base_commit
 
       # Gitaly note: JV: check gitlab-ee before removing this method.
       def rugged_is_ancestor?(ancestor_id, descendant_id)
@@ -774,24 +788,21 @@ module Gitlab
       end
 
       def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
-        OperationService.new(user, self).with_branch(
-          branch_name,
-          start_branch_name: start_branch_name,
-          start_repository: start_repository
-        ) do |start_commit|
-
-          Gitlab::Git.check_namespace!(commit, start_repository)
-
-          revert_tree_id = check_revert_content(commit, start_commit.sha)
-          raise CreateTreeError unless revert_tree_id
-
-          committer = user_to_committer(user)
+        gitaly_migrate(:revert) do |is_enabled|
+          args = {
+            user: user,
+            commit: commit,
+            branch_name: branch_name,
+            message: message,
+            start_branch_name: start_branch_name,
+            start_repository: start_repository
+          }
 
-          create_commit(message: message,
-                        author: committer,
-                        committer: committer,
-                        tree: revert_tree_id,
-                        parents: [start_commit.sha])
+          if is_enabled
+            gitaly_operations_client.user_revert(args)
+          else
+            rugged_revert(args)
+          end
         end
       end
 
@@ -809,44 +820,24 @@ module Gitlab
       end
 
       def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
-        OperationService.new(user, self).with_branch(
-          branch_name,
-          start_branch_name: start_branch_name,
-          start_repository: start_repository
-        ) do |start_commit|
-
-          Gitlab::Git.check_namespace!(commit, start_repository)
-
-          cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha)
-          raise CreateTreeError unless cherry_pick_tree_id
-
-          committer = user_to_committer(user)
+        gitaly_migrate(:cherry_pick) do |is_enabled|
+          args = {
+            user: user,
+            commit: commit,
+            branch_name: branch_name,
+            message: message,
+            start_branch_name: start_branch_name,
+            start_repository: start_repository
+          }
 
-          create_commit(message: message,
-                        author: {
-                            email: commit.author_email,
-                            name: commit.author_name,
-                            time: commit.authored_date
-                        },
-                        committer: committer,
-                        tree: cherry_pick_tree_id,
-                        parents: [start_commit.sha])
+          if is_enabled
+            gitaly_operations_client.user_cherry_pick(args)
+          else
+            rugged_cherry_pick(args)
+          end
         end
       end
 
-      def check_cherry_pick_content(target_commit, source_sha)
-        args = [target_commit.sha, source_sha]
-        args << 1 if target_commit.merge_commit?
-
-        cherry_pick_index = rugged.cherrypick_commit(*args)
-        return false if cherry_pick_index.conflicts?
-
-        tree_id = cherry_pick_index.write_tree(rugged)
-        return false unless diff_exists?(source_sha, tree_id)
-
-        tree_id
-      end
-
       def diff_exists?(sha1, sha2)
         rugged.diff(sha1, sha2).size > 0
       end
@@ -904,8 +895,11 @@ module Gitlab
         end
       end
 
-      def add_remote(remote_name, url)
+      # If `mirror_refmap` is present the remote is set as mirror with that mapping
+      def add_remote(remote_name, url, mirror_refmap: nil)
         rugged.remotes.create(remote_name, url)
+
+        set_remote_as_mirror(remote_name, refmap: mirror_refmap) if mirror_refmap
       rescue Rugged::ConfigError
         remote_update(remote_name, url: url)
       end
@@ -1025,7 +1019,7 @@ module Gitlab
         Gitlab::Git.check_namespace!(start_repository)
         start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository)
 
-        return yield nil if start_repository.empty_repo?
+        return yield nil if start_repository.empty?
 
         if start_repository.same_repository?(self)
           yield commit(start_branch_name)
@@ -1086,17 +1080,17 @@ module Gitlab
         end
       end
 
-      def write_ref(ref_path, ref)
+      def write_ref(ref_path, ref, force: false)
         raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ')
         raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00")
 
-        command = [Gitlab.config.git.bin_path] + %w[update-ref --stdin -z]
-        input = "update #{ref_path}\x00#{ref}\x00\x00"
-        output, status = circuit_breaker.perform do
-          popen(command, path) { |stdin| stdin.write(input) }
-        end
+        ref = "refs/heads/#{ref}" unless ref.start_with?("refs") || ref =~ /\A[a-f0-9]+\z/i
 
-        raise GitError, output unless status.zero?
+        rugged.references.create(ref_path, ref, force: force)
+      rescue Rugged::ReferenceError => ex
+        raise GitError, "could not create ref #{ref_path}: #{ex}"
+      rescue Rugged::OSError => ex
+        raise GitError, "could not create ref #{ref_path}: #{ex}"
       end
 
       def fetch_ref(source_repository, source_ref:, target_ref:)
@@ -1118,12 +1112,22 @@ module Gitlab
       end
 
       # Refactoring aid; allows us to copy code from app/models/repository.rb
-      def run_git(args, env: {})
+      def run_git(args, chdir: path, env: {}, nice: false, &block)
+        cmd = [Gitlab.config.git.bin_path, *args]
+        cmd.unshift("nice") if nice
         circuit_breaker.perform do
-          popen([Gitlab.config.git.bin_path, *args], path, env)
+          popen(cmd, chdir, env, &block)
         end
       end
 
+      def run_git!(args, chdir: path, env: {}, nice: false, &block)
+        output, status = run_git(args, chdir: chdir, env: env, nice: nice, &block)
+
+        raise GitError, output unless status.zero?
+
+        output
+      end
+
       # Refactoring aid; allows us to copy code from app/models/repository.rb
       def run_git_with_timeout(args, timeout, env: {})
         circuit_breaker.perform do
@@ -1136,32 +1140,28 @@ module Gitlab
         Gitlab::Git::Commit.find(self, ref)
       end
 
-      # Refactoring aid; allows us to copy code from app/models/repository.rb
-      def empty_repo?
-        !exists? || !has_visible_content?
+      def empty?
+        !has_visible_content?
       end
 
-      #
-      # Git repository can contains some hidden refs like:
-      #   /refs/notes/*
-      #   /refs/git-as-svn/*
-      #   /refs/pulls/*
-      # This refs by default not visible in project page and not cloned to client side.
-      #
-      # This method return true if repository contains some content visible in project page.
-      #
-      def has_visible_content?
-        return @has_visible_content if defined?(@has_visible_content)
+      def fetch_repository_as_mirror(repository)
+        remote_name = "tmp-#{SecureRandom.hex}"
 
-        @has_visible_content = has_local_branches?
-      end
+        # Notice that this feature flag is not for `fetch_repository_as_mirror`
+        # as a whole but for the fetching mechanism (file path or gitaly-ssh).
+        url, env = gitaly_migrate(:fetch_internal) do |is_enabled|
+          if is_enabled
+            repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository)
+            [GITALY_INTERNAL_URL, repository.fetch_env]
+          else
+            [repository.path, nil]
+          end
+        end
 
-      # Like all public `Gitlab::Git::Repository` methods, this method is part
-      # of `Repository`'s interface through `method_missing`.
-      # `Repository` has its own `fetch_remote` which uses `gitlab-shell` and
-      # takes some extra attributes, so we qualify this method name to prevent confusion.
-      def fetch_remote_without_shell(remote = 'origin')
-        run_git(['fetch', remote]).last.zero?
+        add_remote(remote_name, url, mirror_refmap: :all_refs)
+        fetch_remote(remote_name, env: env)
+      ensure
+        remove_remote(remote_name)
       end
 
       def blob_at(sha, path)
@@ -1187,6 +1187,76 @@ module Gitlab
         end
       end
 
+      def fsck
+        gitaly_migrate(:git_fsck) do |is_enabled|
+          msg, status = if is_enabled
+                          gitaly_fsck
+                        else
+                          shell_fsck
+                        end
+
+          raise GitError.new("Could not fsck repository: #{msg}") unless status.zero?
+        end
+      end
+
+      def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:)
+        rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)
+        env = git_env_for_user(user)
+
+        with_worktree(rebase_path, branch, env: env) do
+          run_git!(
+            %W(pull --rebase #{remote_repository.path} #{remote_branch}),
+            chdir: rebase_path, env: env
+          )
+
+          rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip
+
+          Gitlab::Git::OperationService.new(user, self)
+            .update_branch(branch, rebase_sha, branch_sha)
+
+          rebase_sha
+        end
+      end
+
+      def rebase_in_progress?(rebase_id)
+        fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id))
+      end
+
+      def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:)
+        squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)
+        env = git_env_for_user(user).merge(
+          'GIT_AUTHOR_NAME' => author.name,
+          'GIT_AUTHOR_EMAIL' => author.email
+        )
+        diff_range = "#{start_sha}...#{end_sha}"
+        diff_files = run_git!(
+          %W(diff --name-only --diff-filter=a --binary #{diff_range})
+        ).chomp
+
+        with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do
+          # Apply diff of the `diff_range` to the worktree
+          diff = run_git!(%W(diff --binary #{diff_range}))
+          run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin|
+            stdin.write(diff)
+          end
+
+          # Commit the `diff_range` diff
+          run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env)
+
+          # Return the squash sha. May print a warning for ambiguous refs, but
+          # we can ignore that with `--quiet` and just take the SHA, if present.
+          # HEAD here always refers to the current HEAD commit, even if there is
+          # another ref called HEAD.
+          run_git!(
+            %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env
+          ).chomp
+        end
+      end
+
+      def squash_in_progress?(squash_id)
+        fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id))
+      end
+
       def gitaly_repository
         Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository)
       end
@@ -1223,6 +1293,65 @@ module Gitlab
 
       private
 
+      def fresh_worktree?(path)
+        File.exist?(path) && !clean_stuck_worktree(path)
+      end
+
+      def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:)
+        base_args = %w(worktree add --detach)
+
+        # Note that we _don't_ want to test for `.present?` here: If the caller
+        # passes an non nil empty value it means it still wants sparse checkout
+        # but just isn't interested in any file, perhaps because it wants to
+        # checkout files in by a changeset but that changeset only adds files.
+        if sparse_checkout_files
+          # Create worktree without checking out
+          run_git!(base_args + ['--no-checkout', worktree_path], env: env)
+          worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path)
+
+          configure_sparse_checkout(worktree_git_path, sparse_checkout_files)
+
+          # After sparse checkout configuration, checkout `branch` in worktree
+          run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env)
+        else
+          # Create worktree and checkout `branch` in it
+          run_git!(base_args + [worktree_path, branch], env: env)
+        end
+
+        yield
+      ensure
+        FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path)
+        FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path)
+      end
+
+      def clean_stuck_worktree(path)
+        return false unless File.mtime(path) < 15.minutes.ago
+
+        FileUtils.rm_rf(path)
+        true
+      end
+
+      # Adding a worktree means checking out the repository. For large repos,
+      # this can be very expensive, so set up sparse checkout for the worktree
+      # to only check out the files we're interested in.
+      def configure_sparse_checkout(worktree_git_path, files)
+        run_git!(%w(config core.sparseCheckout true))
+
+        return if files.empty?
+
+        worktree_info_path = File.join(worktree_git_path, 'info')
+        FileUtils.mkdir_p(worktree_info_path)
+        File.write(File.join(worktree_info_path, 'sparse-checkout'), files)
+      end
+
+      def gitaly_fsck
+        gitaly_repository_client.fsck
+      end
+
+      def shell_fsck
+        run_git(%W[--git-dir=#{path} fsck], nice: true)
+      end
+
       def rugged_fetch_source_branch(source_repository, source_branch, local_ref)
         with_repo_branch_commit(source_repository, source_branch) do |commit|
           if commit
@@ -1234,6 +1363,24 @@ module Gitlab
         end
       end
 
+      def worktree_path(prefix, id)
+        id = id.to_s
+        raise ArgumentError, "worktree id can't be empty" unless id.present?
+        raise ArgumentError, "worktree id can't contain slashes " if id.include?("/")
+
+        File.join(path, 'gitlab-worktree', "#{prefix}-#{id}")
+      end
+
+      def git_env_for_user(user)
+        {
+          'GIT_COMMITTER_NAME' => user.name,
+          'GIT_COMMITTER_EMAIL' => user.email,
+          'GL_ID' => Gitlab::GlId.gl_id(user),
+          'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL,
+          'GL_REPOSITORY' => gl_repository
+        }
+      end
+
       # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'.
       def branches_filter(filter: nil, sort_by: nil)
         # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464
@@ -1253,7 +1400,11 @@ module Gitlab
 
       # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695
       def git_merged_branch_names(branch_names = [])
-        root_sha = find_branch(root_ref).target
+        return [] unless root_ref
+
+        root_sha = find_branch(root_ref)&.target
+
+        return [] unless root_sha
 
         git_arguments =
           %W[branch --merged #{root_sha}
@@ -1645,6 +1796,28 @@ module Gitlab
         end
       end
 
+      def rugged_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+        OperationService.new(user, self).with_branch(
+          branch_name,
+          start_branch_name: start_branch_name,
+          start_repository: start_repository
+        ) do |start_commit|
+
+          Gitlab::Git.check_namespace!(commit, start_repository)
+
+          revert_tree_id = check_revert_content(commit, start_commit.sha)
+          raise CreateTreeError unless revert_tree_id
+
+          committer = user_to_committer(user)
+
+          create_commit(message: message,
+                        author: committer,
+                        committer: committer,
+                        tree: revert_tree_id,
+                        parents: [start_commit.sha])
+        end
+      end
+
       def gitaly_add_branch(branch_name, user, target)
         gitaly_operation_client.user_create_branch(branch_name, user, target)
       rescue GRPC::FailedPrecondition => ex
@@ -1661,13 +1834,52 @@ module Gitlab
         raise InvalidRef, ex
       end
 
+      def rugged_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+        OperationService.new(user, self).with_branch(
+          branch_name,
+          start_branch_name: start_branch_name,
+          start_repository: start_repository
+        ) do |start_commit|
+
+          Gitlab::Git.check_namespace!(commit, start_repository)
+
+          cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha)
+          raise CreateTreeError unless cherry_pick_tree_id
+
+          committer = user_to_committer(user)
+
+          create_commit(message: message,
+                        author: {
+                            email: commit.author_email,
+                            name: commit.author_name,
+                            time: commit.authored_date
+                        },
+                        committer: committer,
+                        tree: cherry_pick_tree_id,
+                        parents: [start_commit.sha])
+        end
+      end
+
+      def check_cherry_pick_content(target_commit, source_sha)
+        args = [target_commit.sha, source_sha]
+        args << 1 if target_commit.merge_commit?
+
+        cherry_pick_index = rugged.cherrypick_commit(*args)
+        return false if cherry_pick_index.conflicts?
+
+        tree_id = cherry_pick_index.write_tree(rugged)
+        return false unless diff_exists?(source_sha, tree_id)
+
+        tree_id
+      end
+
       def local_fetch_ref(source_path, source_ref:, target_ref:)
         args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref})
         run_git(args)
       end
 
       def gitaly_fetch_ref(source_repository, source_ref:, target_ref:)
-        args = %W(fetch --no-tags -f ssh://gitaly/internal.git #{source_ref}:#{target_ref})
+        args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref})
 
         run_git(args, env: source_repository.fetch_env)
       end
@@ -1687,6 +1899,10 @@ module Gitlab
       rescue Rugged::ReferenceError
         raise ArgumentError, 'Invalid merge source'
       end
+
+      def fetch_remote(remote_name = 'origin', env: nil)
+        run_git(['fetch', remote_name], env: env).last.zero?
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb
index 392bef69e80..effb1f0ca19 100644
--- a/lib/gitlab/git/repository_mirroring.rb
+++ b/lib/gitlab/git/repository_mirroring.rb
@@ -17,33 +17,6 @@ module Gitlab
         rugged.config["remote.#{remote_name}.prune"] = true
       end
 
-      def set_remote_refmap(remote_name, refmap)
-        Array(refmap).each_with_index do |refspec, i|
-          refspec = REFMAPS[refspec] || refspec
-
-          # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it.
-          # To make sure we start from scratch, we set the first using rugged, and use `git` for any others
-          if i == 0
-            rugged.config["remote.#{remote_name}.fetch"] = refspec
-          else
-            run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
-          end
-        end
-      end
-
-      # Like all_refs public `Gitlab::Git::Repository` methods, this method is part
-      # of `Repository`'s interface through `method_missing`.
-      # `Repository` has its own `fetch_as_mirror` which uses `gitlab-shell` and
-      # takes some extra attributes, so we qualify this method name to prevent confusion.
-      def fetch_as_mirror_without_shell(url)
-        remote_name = "tmp-#{SecureRandom.hex}"
-        add_remote(remote_name, url)
-        set_remote_as_mirror(remote_name)
-        fetch_remote_without_shell(remote_name)
-      ensure
-        remove_remote(remote_name) if remote_name
-      end
-
       def remote_tags(remote)
         # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n"
         # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...]
@@ -85,6 +58,20 @@ module Gitlab
 
       private
 
+      def set_remote_refmap(remote_name, refmap)
+        Array(refmap).each_with_index do |refspec, i|
+          refspec = REFMAPS[refspec] || refspec
+
+          # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it.
+          # To make sure we start from scratch, we set the first using rugged, and use `git` for any others
+          if i == 0
+            rugged.config["remote.#{remote_name}.fetch"] = refspec
+          else
+            run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
+          end
+        end
+      end
+
       def list_remote_tags(remote)
         tag_list, exit_code, error = nil
         cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-remote --tags #{remote})
diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb
index 99518c9b1e4..5933312b0b5 100644
--- a/lib/gitlab/git/storage.rb
+++ b/lib/gitlab/git/storage.rb
@@ -15,6 +15,7 @@ module Gitlab
       Failing = Class.new(Inaccessible)
 
       REDIS_KEY_PREFIX = 'storage_accessible:'.freeze
+      REDIS_KNOWN_KEYS = "#{REDIS_KEY_PREFIX}known_keys_set".freeze
 
       def self.redis
         Gitlab::Redis::SharedState
diff --git a/lib/gitlab/git/storage/checker.rb b/lib/gitlab/git/storage/checker.rb
new file mode 100644
index 00000000000..d3c37f82101
--- /dev/null
+++ b/lib/gitlab/git/storage/checker.rb
@@ -0,0 +1,120 @@
+module Gitlab
+  module Git
+    module Storage
+      class Checker
+        include CircuitBreakerSettings
+
+        attr_reader :storage_path, :storage, :hostname, :logger
+        METRICS_MUTEX = Mutex.new
+        STORAGE_TIMING_BUCKETS = [0.1, 0.15, 0.25, 0.33, 0.5, 1, 1.5, 2.5, 5, 10, 15].freeze
+
+        def self.check_all(logger = Rails.logger)
+          threads = Gitlab.config.repositories.storages.keys.map do |storage_name|
+            Thread.new do
+              Thread.current[:result] = new(storage_name, logger).check_with_lease
+            end
+          end
+
+          threads.map do |thread|
+            thread.join
+            thread[:result]
+          end
+        end
+
+        def self.check_histogram
+          @check_histogram ||=
+            METRICS_MUTEX.synchronize do
+              @check_histogram || Gitlab::Metrics.histogram(:circuitbreaker_storage_check_duration_seconds,
+                                                            'Storage check time in seconds',
+                                                            {},
+                                                            STORAGE_TIMING_BUCKETS
+                                                           )
+            end
+        end
+
+        def initialize(storage, logger = Rails.logger)
+          @storage = storage
+          config = Gitlab.config.repositories.storages[@storage]
+          @storage_path = config['path']
+          @logger = logger
+
+          @hostname = Gitlab::Environment.hostname
+        end
+
+        def check_with_lease
+          lease_key = "storage_check:#{cache_key}"
+          lease = Gitlab::ExclusiveLease.new(lease_key, timeout: storage_timeout)
+          result = { storage: storage, success: nil }
+
+          if uuid = lease.try_obtain
+            result[:success] = check
+
+            Gitlab::ExclusiveLease.cancel(lease_key, uuid)
+          else
+            logger.warn("#{hostname}: #{storage}: Skipping check, previous check still running")
+          end
+
+          result
+        end
+
+        def check
+          if perform_access_check
+            track_storage_accessible
+            true
+          else
+            track_storage_inaccessible
+            logger.error("#{hostname}: #{storage}: Not accessible.")
+            false
+          end
+        end
+
+        private
+
+        def perform_access_check
+          start_time = Gitlab::Metrics::System.monotonic_time
+
+          Gitlab::Git::Storage::ForkedStorageCheck.storage_available?(storage_path, storage_timeout, access_retries)
+        ensure
+          execution_time = Gitlab::Metrics::System.monotonic_time - start_time
+          self.class.check_histogram.observe({ storage: storage }, execution_time)
+        end
+
+        def track_storage_inaccessible
+          first_failure = current_failure_info.first_failure || Time.now
+          last_failure = Time.now
+
+          Gitlab::Git::Storage.redis.with do |redis|
+            redis.pipelined do
+              redis.hset(cache_key, :first_failure, first_failure.to_i)
+              redis.hset(cache_key, :last_failure, last_failure.to_i)
+              redis.hincrby(cache_key, :failure_count, 1)
+              redis.expire(cache_key, failure_reset_time)
+              maintain_known_keys(redis)
+            end
+          end
+        end
+
+        def track_storage_accessible
+          Gitlab::Git::Storage.redis.with do |redis|
+            redis.pipelined do
+              redis.hset(cache_key, :first_failure, nil)
+              redis.hset(cache_key, :last_failure, nil)
+              redis.hset(cache_key, :failure_count, 0)
+              maintain_known_keys(redis)
+            end
+          end
+        end
+
+        def maintain_known_keys(redis)
+          expire_time = Time.now.to_i + failure_reset_time
+          redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, expire_time, cache_key)
+          redis.zremrangebyscore(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, '-inf', Time.now.to_i)
+        end
+
+        def current_failure_info
+          FailureInfo.load(cache_key)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb
index be7598ef011..898bb1b65be 100644
--- a/lib/gitlab/git/storage/circuit_breaker.rb
+++ b/lib/gitlab/git/storage/circuit_breaker.rb
@@ -4,24 +4,11 @@ module Gitlab
       class CircuitBreaker
         include CircuitBreakerSettings
 
-        FailureInfo = Struct.new(:last_failure, :failure_count)
-
         attr_reader :storage,
-                    :hostname,
-                    :storage_path
-
-        delegate :last_failure, :failure_count, to: :failure_info
-
-        def self.reset_all!
-          pattern = "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}*"
+                    :hostname
 
-          Gitlab::Git::Storage.redis.with do |redis|
-            all_storage_keys = redis.scan_each(match: pattern).to_a
-            redis.del(*all_storage_keys) unless all_storage_keys.empty?
-          end
-
-          RequestStore.delete(:circuitbreaker_cache)
-        end
+        delegate :last_failure, :failure_count, :no_failures?,
+                 to: :failure_info
 
         def self.for_storage(storage)
           cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do
@@ -48,9 +35,6 @@ module Gitlab
         def initialize(storage, hostname)
           @storage = storage
           @hostname = hostname
-
-          config = Gitlab.config.repositories.storages[@storage]
-          @storage_path = config['path']
         end
 
         def perform
@@ -67,15 +51,6 @@ module Gitlab
           failure_count > failure_count_threshold
         end
 
-        def backing_off?
-          return false if no_failures?
-
-          recent_failure = last_failure > failure_wait_time.seconds.ago
-          too_many_failures = failure_count > backoff_threshold
-
-          recent_failure && too_many_failures
-        end
-
         private
 
         # The circuitbreaker can be enabled for the entire fleet using a Feature
@@ -88,82 +63,13 @@ module Gitlab
         end
 
         def failure_info
-          @failure_info ||= get_failure_info
-        end
-
-        # Memoizing the `storage_available` call means we only do it once per
-        # request when the storage is available.
-        #
-        # When the storage appears not available, and the memoized value is `false`
-        # we might want to try again.
-        def storage_available?
-          return @storage_available if @storage_available
-
-          if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck
-                                    .storage_available?(storage_path, storage_timeout, access_retries)
-            track_storage_accessible
-          else
-            track_storage_inaccessible
-          end
-
-          @storage_available
+          @failure_info ||= FailureInfo.load(cache_key)
         end
 
         def check_storage_accessible!
           if circuit_broken?
             raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time)
           end
-
-          if backing_off?
-            raise Gitlab::Git::Storage::Failing.new("Backing off access to #{storage}", failure_wait_time)
-          end
-
-          unless storage_available?
-            raise Gitlab::Git::Storage::Inaccessible.new("#{storage} not accessible", failure_wait_time)
-          end
-        end
-
-        def no_failures?
-          last_failure.blank? && failure_count == 0
-        end
-
-        def track_storage_inaccessible
-          @failure_info = FailureInfo.new(Time.now, failure_count + 1)
-
-          Gitlab::Git::Storage.redis.with do |redis|
-            redis.pipelined do
-              redis.hset(cache_key, :last_failure, last_failure.to_i)
-              redis.hincrby(cache_key, :failure_count, 1)
-              redis.expire(cache_key, failure_reset_time)
-            end
-          end
-        end
-
-        def track_storage_accessible
-          return if no_failures?
-
-          @failure_info = FailureInfo.new(nil, 0)
-
-          Gitlab::Git::Storage.redis.with do |redis|
-            redis.pipelined do
-              redis.hset(cache_key, :last_failure, nil)
-              redis.hset(cache_key, :failure_count, 0)
-            end
-          end
-        end
-
-        def get_failure_info
-          last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis|
-            redis.hmget(cache_key, :last_failure, :failure_count)
-          end
-
-          last_failure = Time.at(last_failure.to_i) if last_failure.present?
-
-          FailureInfo.new(last_failure, failure_count.to_i)
-        end
-
-        def cache_key
-          @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
         end
       end
     end
diff --git a/lib/gitlab/git/storage/circuit_breaker_settings.rb b/lib/gitlab/git/storage/circuit_breaker_settings.rb
index 257fe8cd8f0..c9e225f187d 100644
--- a/lib/gitlab/git/storage/circuit_breaker_settings.rb
+++ b/lib/gitlab/git/storage/circuit_breaker_settings.rb
@@ -6,10 +6,6 @@ module Gitlab
           application_settings.circuitbreaker_failure_count_threshold
         end
 
-        def failure_wait_time
-          application_settings.circuitbreaker_failure_wait_time
-        end
-
         def failure_reset_time
           application_settings.circuitbreaker_failure_reset_time
         end
@@ -22,8 +18,12 @@ module Gitlab
           application_settings.circuitbreaker_access_retries
         end
 
-        def backoff_threshold
-          application_settings.circuitbreaker_backoff_threshold
+        def check_interval
+          application_settings.circuitbreaker_check_interval
+        end
+
+        def cache_key
+          @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
         end
 
         private
diff --git a/lib/gitlab/git/storage/failure_info.rb b/lib/gitlab/git/storage/failure_info.rb
new file mode 100644
index 00000000000..387279c110d
--- /dev/null
+++ b/lib/gitlab/git/storage/failure_info.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module Git
+    module Storage
+      class FailureInfo
+        attr_accessor :first_failure, :last_failure, :failure_count
+
+        def self.reset_all!
+          Gitlab::Git::Storage.redis.with do |redis|
+            all_storage_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1)
+            redis.del(*all_storage_keys) unless all_storage_keys.empty?
+          end
+
+          RequestStore.delete(:circuitbreaker_cache)
+        end
+
+        def self.load(cache_key)
+          first_failure, last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis|
+            redis.hmget(cache_key, :first_failure, :last_failure, :failure_count)
+          end
+
+          last_failure = Time.at(last_failure.to_i) if last_failure.present?
+          first_failure = Time.at(first_failure.to_i) if first_failure.present?
+
+          new(first_failure, last_failure, failure_count.to_i)
+        end
+
+        def initialize(first_failure, last_failure, failure_count)
+          @first_failure = first_failure
+          @last_failure = last_failure
+          @failure_count = failure_count
+        end
+
+        def no_failures?
+          first_failure.blank? && last_failure.blank? && failure_count == 0
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb
index 7049772fe3b..90bbe85fd37 100644
--- a/lib/gitlab/git/storage/health.rb
+++ b/lib/gitlab/git/storage/health.rb
@@ -4,8 +4,8 @@ module Gitlab
       class Health
         attr_reader :storage_name, :info
 
-        def self.pattern_for_storage(storage_name)
-          "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage_name}:*"
+        def self.prefix_for_storage(storage_name)
+          "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage_name}:"
         end
 
         def self.for_all_storages
@@ -25,26 +25,15 @@ module Gitlab
 
         private_class_method def self.all_keys_for_storages(storage_names, redis)
           keys_per_storage = {}
+          all_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1)
 
-          redis.pipelined do
-            storage_names.each do |storage_name|
-              pattern = pattern_for_storage(storage_name)
-              matched_keys = redis.scan_each(match: pattern)
+          storage_names.each do |storage_name|
+            prefix = prefix_for_storage(storage_name)
 
-              keys_per_storage[storage_name] = matched_keys
-            end
+            keys_per_storage[storage_name] = all_keys.select { |key| key.starts_with?(prefix) }
           end
 
-          # We need to make sure each lazy-loaded `Enumerator` for matched keys
-          # is loaded into an array.
-          #
-          # Otherwise it would be loaded in the second `Redis#pipelined` block
-          # within `.load_for_keys`. In this pipelined call, the active
-          # Redis-client changes again, so the values would not be available
-          # until the end of that pipelined-block.
-          keys_per_storage.each do |storage_name, key_future|
-            keys_per_storage[storage_name] = key_future.to_a
-          end
+          keys_per_storage
         end
 
         private_class_method def self.load_for_keys(keys_per_storage, redis)
diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb
index a12d52d295f..261c936c689 100644
--- a/lib/gitlab/git/storage/null_circuit_breaker.rb
+++ b/lib/gitlab/git/storage/null_circuit_breaker.rb
@@ -11,6 +11,9 @@ module Gitlab
         # These will always have nil values
         attr_reader :storage_path
 
+        delegate :last_failure, :failure_count, :no_failures?,
+                 to: :failure_info
+
         def initialize(storage, hostname, error: nil)
           @storage = storage
           @hostname = hostname
@@ -29,16 +32,17 @@ module Gitlab
           false
         end
 
-        def last_failure
-          circuit_broken? ? Time.now : nil
-        end
-
-        def failure_count
-          circuit_broken? ? failure_count_threshold : 0
-        end
-
         def failure_info
-          Gitlab::Git::Storage::CircuitBreaker::FailureInfo.new(last_failure, failure_count)
+          @failure_info ||=
+            if circuit_broken?
+              Gitlab::Git::Storage::FailureInfo.new(Time.now,
+                                                    Time.now,
+                                                    failure_count_threshold)
+            else
+              Gitlab::Git::Storage::FailureInfo.new(nil,
+                                                    nil,
+                                                    0)
+            end
         end
       end
     end
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 8998c4b1a83..56f6febe86d 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -102,18 +102,15 @@ module Gitlab
     end
 
     def check_project_moved!
-      return unless redirected_path
+      return if redirected_path.nil?
 
-      url = protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo
-      message = <<-MESSAGE.strip_heredoc
-        Project '#{redirected_path}' was moved to '#{project.full_path}'.
+      project_moved = Checks::ProjectMoved.new(project, user, redirected_path, protocol)
 
-        Please update your Git remote and try again:
-
-          git remote set-url origin #{url}
-      MESSAGE
-
-      raise ProjectMovedError, message
+      if project_moved.permanent_redirect?
+        project_moved.add_redirect_message
+      else
+        raise ProjectMovedError, project_moved.redirect_message(rejected: true)
+      end
     end
 
     def check_command_disabled!(cmd)
@@ -166,7 +163,7 @@ module Gitlab
       end
 
       if Gitlab::Database.read_only?
-        raise UnauthorizedError, ERROR_MESSAGES[:cannot_push_to_read_only]
+        raise UnauthorizedError, push_to_read_only_message
       end
 
       if deploy_key
@@ -280,5 +277,9 @@ module Gitlab
                          UserAccess.new(user, project: project)
                        end
     end
+
+    def push_to_read_only_message
+      ERROR_MESSAGES[:cannot_push_to_read_only]
+    end
   end
 end
diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb
index 98f1f45b338..1c9477e84b2 100644
--- a/lib/gitlab/git_access_wiki.rb
+++ b/lib/gitlab/git_access_wiki.rb
@@ -19,10 +19,14 @@ module Gitlab
       end
 
       if Gitlab::Database.read_only?
-        raise UnauthorizedError, ERROR_MESSAGES[:read_only]
+        raise UnauthorizedError, push_to_read_only_message
       end
 
       true
     end
+
+    def push_to_read_only_message
+      ERROR_MESSAGES[:read_only]
+    end
   end
 end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index f27cd800bdd..b753ac46291 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -27,7 +27,7 @@ module Gitlab
     end
 
     SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze
-    MAXIMUM_GITALY_CALLS = 30
+    MAXIMUM_GITALY_CALLS = 35
     CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
 
     MUTEX = Mutex.new
@@ -336,6 +336,12 @@ module Gitlab
       s.dup.force_encoding(Encoding::ASCII_8BIT)
     end
 
+    def self.binary_stringio(s)
+      io = StringIO.new(s || '')
+      io.set_encoding(Encoding::ASCII_8BIT)
+      io
+    end
+
     def self.encode_repeated(a)
       Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } )
     end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 34807d280e5..7985f5b5457 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -250,6 +250,26 @@ module Gitlab
         consume_commits_response(response)
       end
 
+      def filter_shas_with_signatures(shas)
+        request = Gitaly::FilterShasWithSignaturesRequest.new(repository: @gitaly_repo)
+
+        enum = Enumerator.new do |y|
+          shas.each_slice(20) do |revs|
+            request.shas = GitalyClient.encode_repeated(revs)
+
+            y.yield request
+
+            request = Gitaly::FilterShasWithSignaturesRequest.new
+          end
+        end
+
+        response = GitalyClient.call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum)
+
+        response.flat_map do |msg|
+          msg.shas.map { |sha| EncodingHelper.encode!(sha) }
+        end
+      end
+
       private
 
       def call_commit_diff(request_params, options = {})
diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb
index 526d44a8b77..400a4af363b 100644
--- a/lib/gitlab/gitaly_client/operation_service.rb
+++ b/lib/gitlab/gitaly_client/operation_service.rb
@@ -122,6 +122,64 @@ module Gitlab
         ).branch_update
         Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update)
       end
+
+      def user_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+        call_cherry_pick_or_revert(:cherry_pick,
+                                   user: user,
+                                   commit: commit,
+                                   branch_name: branch_name,
+                                   message: message,
+                                   start_branch_name: start_branch_name,
+                                   start_repository: start_repository)
+      end
+
+      def user_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+        call_cherry_pick_or_revert(:revert,
+                                   user: user,
+                                   commit: commit,
+                                   branch_name: branch_name,
+                                   message: message,
+                                   start_branch_name: start_branch_name,
+                                   start_repository: start_repository)
+      end
+
+      private
+
+      def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+        request_class = "Gitaly::User#{rpc.to_s.camelcase}Request".constantize
+
+        request = request_class.new(
+          repository: @gitaly_repo,
+          user: Gitlab::Git::User.from_gitlab(user).to_gitaly,
+          commit: commit.to_gitaly_commit,
+          branch_name: GitalyClient.encode(branch_name),
+          message: GitalyClient.encode(message),
+          start_branch_name: GitalyClient.encode(start_branch_name.to_s),
+          start_repository: start_repository.gitaly_repository
+        )
+
+        response = GitalyClient.call(
+          @repository.storage,
+          :operation_service,
+          :"user_#{rpc}",
+          request,
+          remote_storage: start_repository.storage
+        )
+
+        handle_cherry_pick_or_revert_response(response)
+      end
+
+      def handle_cherry_pick_or_revert_response(response)
+        if response.pre_receive_error.presence
+          raise Gitlab::Git::HooksService::PreReceiveError, response.pre_receive_error
+        elsif response.commit_error.presence
+          raise Gitlab::Git::CommitError, response.commit_error
+        elsif response.create_tree_error.presence
+          raise Gitlab::Git::Repository::CreateTreeError, response.create_tree_error
+        else
+          Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index b9e606592d7..c1f95396878 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -69,6 +69,16 @@ module Gitlab
         response.value
       end
 
+      def find_merge_base(*revisions)
+        request = Gitaly::FindMergeBaseRequest.new(
+          repository: @gitaly_repo,
+          revisions: revisions.map { |r| GitalyClient.encode(r) }
+        )
+
+        response = GitalyClient.call(@storage, :repository_service, :find_merge_base, request)
+        response.base.presence
+      end
+
       def fetch_source_branch(source_repository, source_branch, local_ref)
         request = Gitaly::FetchSourceBranchRequest.new(
           repository: @gitaly_repo,
@@ -87,6 +97,17 @@ module Gitlab
 
         response.result
       end
+
+      def fsck
+        request = Gitaly::FsckRequest.new(repository: @gitaly_repo)
+        response = GitalyClient.call(@storage, :repository_service, :fsck, request)
+
+        if response.error.empty?
+          return "", 0
+        else
+          return response.error.b, 1
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb
index c8f065f5881..337d225d081 100644
--- a/lib/gitlab/gitaly_client/wiki_service.rb
+++ b/lib/gitlab/gitaly_client/wiki_service.rb
@@ -18,12 +18,11 @@ module Gitlab
           commit_details: gitaly_commit_details(commit_details)
         )
 
-        strio = StringIO.new(content)
+        strio = GitalyClient.binary_stringio(content)
 
         enum = Enumerator.new do |y|
           until strio.eof?
-            chunk = strio.read(MAX_MSG_SIZE)
-            request.content = GitalyClient.encode(chunk)
+            request.content = strio.read(MAX_MSG_SIZE)
 
             y.yield request
 
@@ -46,12 +45,11 @@ module Gitlab
           commit_details: gitaly_commit_details(commit_details)
         )
 
-        strio = StringIO.new(content)
+        strio = GitalyClient.binary_stringio(content)
 
         enum = Enumerator.new do |y|
           until strio.eof?
-            chunk = strio.read(MAX_MSG_SIZE)
-            request.content = GitalyClient.encode(chunk)
+            request.content = strio.read(MAX_MSG_SIZE)
 
             y.yield request
 
diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb
index 94678b6ec40..3f3f10596c5 100644
--- a/lib/gitlab/identifier.rb
+++ b/lib/gitlab/identifier.rb
@@ -2,9 +2,8 @@
 # key-13 or user-36 or last commit
 module Gitlab
   module Identifier
-    def identify(identifier, project, newrev)
+    def identify(identifier, project = nil, newrev = nil)
       if identifier.blank?
-        # Local push from gitlab
         identify_using_commit(project, newrev)
       elsif identifier =~ /\Auser-\d+\Z/
         # git push over http
@@ -17,6 +16,8 @@ module Gitlab
 
     # Tries to identify a user based on a commit SHA.
     def identify_using_commit(project, ref)
+      return if project.nil? && ref.nil?
+
       commit = project.commit(ref)
 
       return if !commit || !commit.author_email
diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb
index 7a50f07f3c5..407cdefc04d 100644
--- a/lib/gitlab/kubernetes/helm.rb
+++ b/lib/gitlab/kubernetes/helm.rb
@@ -18,7 +18,7 @@ module Gitlab
 
       def initialize(kubeclient)
         @kubeclient = kubeclient
-        @namespace = Namespace.new(NAMESPACE, kubeclient)
+        @namespace = Gitlab::Kubernetes::Namespace.new(NAMESPACE, kubeclient)
       end
 
       def install(command)
diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb
index 65d55576ac2..9112164f22e 100644
--- a/lib/gitlab/metrics/method_call.rb
+++ b/lib/gitlab/metrics/method_call.rb
@@ -1,7 +1,11 @@
+# rubocop:disable Style/ClassVars
+
 module Gitlab
   module Metrics
     # Class for tracking timing information about method calls
     class MethodCall
+      @@measurement_enabled_cache = Concurrent::AtomicBoolean.new(false)
+      @@measurement_enabled_cache_expires_at = Concurrent::AtomicFixnum.new(Time.now.to_i)
       MUTEX = Mutex.new
       BASE_LABELS = { module: nil, method: nil }.freeze
       attr_reader :real_time, :cpu_time, :call_count, :labels
@@ -18,6 +22,10 @@ module Gitlab
         end
       end
 
+      def self.measurement_enabled_cache_expires_at
+        @@measurement_enabled_cache_expires_at
+      end
+
       # name - The full name of the method (including namespace) such as
       #        `User#sign_in`.
       #
@@ -72,7 +80,14 @@ module Gitlab
       end
 
       def call_measurement_enabled?
-        Feature.get(:prometheus_metrics_method_instrumentation).enabled?
+        expires_at = @@measurement_enabled_cache_expires_at.value
+        if expires_at < Time.now.to_i
+          if @@measurement_enabled_cache_expires_at.compare_and_set(expires_at, 1.minute.from_now.to_i)
+            @@measurement_enabled_cache.value = Feature.get(:prometheus_metrics_method_instrumentation).enabled?
+          end
+        end
+
+        @@measurement_enabled_cache.value
       end
     end
   end
diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb
index f4f9b5ca792..5a0f7f28fc8 100644
--- a/lib/gitlab/metrics/samplers/influx_sampler.rb
+++ b/lib/gitlab/metrics/samplers/influx_sampler.rb
@@ -27,7 +27,6 @@ module Gitlab
         def sample
           sample_memory_usage
           sample_file_descriptors
-          sample_objects
           sample_gc
 
           flush
@@ -48,29 +47,6 @@ module Gitlab
           add_metric('file_descriptors', value: System.file_descriptor_count)
         end
 
-        if Metrics.mri?
-          def sample_objects
-            sample = Allocations.to_hash
-            counts = sample.each_with_object({}) do |(klass, count), hash|
-              name = klass.name
-
-              next unless name
-
-              hash[name] = count
-            end
-
-            # Symbols aren't allocated so we'll need to add those manually.
-            counts['Symbol'] = Symbol.all_symbols.length
-
-            counts.each do |name, count|
-              add_metric('object_counts', { count: count }, type: name)
-            end
-          end
-        else
-          def sample_objects
-          end
-        end
-
         def sample_gc
           time = GC::Profiler.total_time * 1000.0
           stats = GC.stat.merge(total_time: time)
diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb
index 436a9e9550d..b68800417a2 100644
--- a/lib/gitlab/metrics/samplers/ruby_sampler.rb
+++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb
@@ -32,7 +32,7 @@ module Gitlab
 
         def init_metrics
           metrics = {}
-          metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', {})
+          metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', { worker: nil })
           metrics[:total_time] = Metrics.gauge(with_prefix(:gc, :time_total), 'Total GC time', labels, :livesum)
           GC.stat.keys.each do |key|
             metrics[key] = Metrics.gauge(with_prefix(:gc, key), to_doc_string(key), labels, :livesum)
@@ -48,7 +48,6 @@ module Gitlab
         def sample
           start_time = System.monotonic_time
           sample_gc
-          sample_objects
 
           metrics[:memory_usage].set(labels, System.memory_usage)
           metrics[:file_descriptors].set(labels, System.file_descriptor_count)
@@ -68,41 +67,15 @@ module Gitlab
           end
         end
 
-        def sample_objects
-          list_objects.each do |name, count|
-            metrics[:objects_total].set(labels.merge(class: name), count)
-          end
-        end
-
-        if Metrics.mri?
-          def list_objects
-            sample = Allocations.to_hash
-            counts = sample.each_with_object({}) do |(klass, count), hash|
-              name = klass.name
-
-              next unless name
-
-              hash[name] = count
-            end
-
-            # Symbols aren't allocated so we'll need to add those manually.
-            counts['Symbol'] = Symbol.all_symbols.length
-            counts
-          end
-        else
-          def list_objects
-          end
-        end
-
         def worker_label
           return {} unless defined?(Unicorn::Worker)
 
           worker_no = ::Prometheus::Client::Support::Unicorn.worker_id
 
           if worker_no
-            { unicorn: worker_no }
+            { worker: worker_no }
           else
-            { unicorn: 'master' }
+            { worker: 'master' }
           end
         end
       end
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 561aa9e162c..e2662fc362b 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -47,8 +47,11 @@ module Gitlab
       startline = 0
 
       result.each_line.each_with_index do |line, index|
-        if line =~ /^.*:.*:\d+:/
-          ref, filename, startline = line.split(':')
+        matches = line.match(/^(?<ref>[^:]*):(?<filename>.*):(?<startline>\d+):/)
+        if matches
+          ref = matches[:ref]
+          filename = matches[:filename]
+          startline = matches[:startline]
           startline = startline.to_i - index
           extname = Regexp.escape(File.extname(filename))
           basename = filename.sub(/#{extname}$/, '')
diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
index 7ac6162b54d..5cddc96a643 100644
--- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb
+++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
@@ -76,7 +76,7 @@ module Gitlab
             timeframe_start: timeframe_start,
             timeframe_end: timeframe_end,
             ci_environment_slug: environment.slug,
-            kube_namespace: environment.project.kubernetes_service&.actual_namespace || '',
+            kube_namespace: environment.project.deployment_platform&.actual_namespace || '',
             environment_filter: %{container_name!="POD",environment="#{environment.slug}"}
           }
         end
diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb
index bc836dcc08d..9ff82d628c0 100644
--- a/lib/gitlab/reference_extractor.rb
+++ b/lib/gitlab/reference_extractor.rb
@@ -1,7 +1,7 @@
 module Gitlab
   # Extract possible GFM references from an arbitrary String for further processing.
   class ReferenceExtractor < Banzai::ReferenceExtractor
-    REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user).freeze
+    REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user epic).freeze
     attr_accessor :project, :current_user, :author
 
     def initialize(project, current_user = nil)
diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb
index f9ab9bd466f..30df7e4a831 100644
--- a/lib/gitlab/seeder.rb
+++ b/lib/gitlab/seeder.rb
@@ -8,7 +8,8 @@ end
 module Gitlab
   class Seeder
     def self.quiet
-      mute_mailer
+      mute_mailer unless Rails.env.test?
+
       SeedFu.quiet = true
 
       yield
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index 996d213fdb4..a22a63665be 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -143,20 +143,27 @@ module Gitlab
                                  storage, "#{path}.git", "#{new_path}.git"])
     end
 
-    # Fork repository to new namespace
+    # Fork repository to new path
     # forked_from_storage - forked-from project's storage path
-    # path - project path with namespace
+    # forked_from_disk_path - project disk path
     # forked_to_storage - forked-to project's storage path
-    # fork_namespace - namespace for forked project
+    # forked_to_disk_path - forked project disk path
     #
     # Ex.
-    #  fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx")
+    #  fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci")
     #
     # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one.
-    def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace)
-      gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project',
-                                 forked_from_storage, "#{path}.git", forked_to_storage,
-                                 fork_namespace])
+    def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path)
+      gitlab_shell_fast_execute(
+        [
+          gitlab_shell_projects_path,
+          'fork-repository',
+          forked_from_storage,
+          "#{forked_from_disk_path}.git",
+          forked_to_storage,
+          "#{forked_to_disk_path}.git"
+        ]
+      )
     end
 
     # Remove repository from file system
diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb
new file mode 100644
index 00000000000..c3d7814551c
--- /dev/null
+++ b/lib/gitlab/sidekiq_config.rb
@@ -0,0 +1,61 @@
+require 'yaml'
+require 'set'
+
+module Gitlab
+  module SidekiqConfig
+    # This method is called by `bin/sidekiq-cluster` in EE, which runs outside
+    # of bundler/Rails context, so we cannot use any gem or Rails methods.
+    def self.worker_queues(rails_path = Rails.root.to_s)
+      @worker_queues ||= {}
+      @worker_queues[rails_path] ||= YAML.load_file(File.join(rails_path, 'app/workers/all_queues.yml'))
+    end
+
+    # This method is called by `bin/sidekiq-cluster` in EE, which runs outside
+    # of bundler/Rails context, so we cannot use any gem or Rails methods.
+    def self.expand_queues(queues, all_queues = self.worker_queues)
+      return [] if queues.empty?
+
+      queues_set = all_queues.to_set
+
+      queues.flat_map do |queue|
+        [queue, *queues_set.grep(/\A#{queue}:/)]
+      end
+    end
+
+    def self.redis_queues
+      # Not memoized, because this can change during the life of the application
+      Sidekiq::Queue.all.map(&:name)
+    end
+
+    def self.config_queues
+      @config_queues ||= begin
+        config = YAML.load_file(Rails.root.join('config/sidekiq_queues.yml'))
+        config[:queues].map(&:first)
+      end
+    end
+
+    def self.cron_workers
+      @cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize }
+    end
+
+    def self.workers
+      @workers ||= find_workers(Rails.root.join('app', 'workers'))
+    end
+
+    def self.find_workers(root)
+      concerns = root.join('concerns').to_s
+
+      workers = Dir[root.join('**', '*.rb')]
+        .reject { |path| path.start_with?(concerns) }
+
+      workers.map! do |path|
+        ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '')
+
+        ns.camelize.constantize
+      end
+
+      # Skip things that aren't workers
+      workers.select { |w| w < Sidekiq::Worker }
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_versioning.rb b/lib/gitlab/sidekiq_versioning.rb
new file mode 100644
index 00000000000..9683214ec18
--- /dev/null
+++ b/lib/gitlab/sidekiq_versioning.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module SidekiqVersioning
+    def self.install!
+      Sidekiq::Manager.prepend SidekiqVersioning::Manager
+
+      # The Sidekiq client API always adds the queue to the Sidekiq queue
+      # list, but mail_room and gitlab-shell do not. This is only necessary
+      # for monitoring.
+      begin
+        queues = SidekiqConfig.worker_queues
+
+        if queues.any?
+          Sidekiq.redis do |conn|
+            conn.pipelined do
+              queues.each do |queue|
+                conn.sadd('queues', queue)
+              end
+            end
+          end
+        end
+      rescue ::Redis::BaseError, SocketError, Errno::ENOENT, Errno::EADDRNOTAVAIL, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_versioning/manager.rb b/lib/gitlab/sidekiq_versioning/manager.rb
new file mode 100644
index 00000000000..308be0fdf76
--- /dev/null
+++ b/lib/gitlab/sidekiq_versioning/manager.rb
@@ -0,0 +1,12 @@
+module Gitlab
+  module SidekiqVersioning
+    module Manager
+      def initialize(options = {})
+        options[:strict] = false
+        options[:queues] = SidekiqConfig.expand_queues(options[:queues])
+        Sidekiq.logger.info "Listening on queues #{options[:queues].uniq.sort}"
+        super
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/storage_check.rb b/lib/gitlab/storage_check.rb
new file mode 100644
index 00000000000..fe81513c9ec
--- /dev/null
+++ b/lib/gitlab/storage_check.rb
@@ -0,0 +1,11 @@
+require_relative 'storage_check/cli'
+require_relative 'storage_check/gitlab_caller'
+require_relative 'storage_check/option_parser'
+require_relative 'storage_check/response'
+
+module Gitlab
+  module StorageCheck
+    ENDPOINT = '/-/storage_check'.freeze
+    Options = Struct.new(:target, :token, :interval, :dryrun)
+  end
+end
diff --git a/lib/gitlab/storage_check/cli.rb b/lib/gitlab/storage_check/cli.rb
new file mode 100644
index 00000000000..04bf1bf1d26
--- /dev/null
+++ b/lib/gitlab/storage_check/cli.rb
@@ -0,0 +1,69 @@
+module Gitlab
+  module StorageCheck
+    class CLI
+      def self.start!(args)
+        runner = new(Gitlab::StorageCheck::OptionParser.parse!(args))
+        runner.start_loop
+      end
+
+      attr_reader :logger, :options
+
+      def initialize(options)
+        @options = options
+        @logger = Logger.new(STDOUT)
+      end
+
+      def start_loop
+        logger.info "Checking #{options.target} every #{options.interval} seconds"
+
+        if options.dryrun
+          logger.info "Dryrun, exiting..."
+          return
+        end
+
+        begin
+          loop do
+            response = GitlabCaller.new(options).call!
+            log_response(response)
+            update_settings(response)
+
+            sleep options.interval
+          end
+        rescue Interrupt
+          logger.info "Ending storage-check"
+        end
+      end
+
+      def update_settings(response)
+        previous_interval = options.interval
+
+        if response.valid?
+          options.interval = response.check_interval || previous_interval
+        end
+
+        if previous_interval != options.interval
+          logger.info "Interval changed: #{options.interval} seconds"
+        end
+      end
+
+      def log_response(response)
+        unless response.valid?
+          return logger.error("Invalid response checking nfs storage: #{response.http_response.inspect}")
+        end
+
+        if response.responsive_shards.any?
+          logger.debug("Responsive shards: #{response.responsive_shards.join(', ')}")
+        end
+
+        warnings = []
+        if response.skipped_shards.any?
+          warnings << "Skipped shards: #{response.skipped_shards.join(', ')}"
+        end
+        if response.failing_shards.any?
+          warnings << "Failing shards: #{response.failing_shards.join(', ')}"
+        end
+        logger.warn(warnings.join(' - ')) if warnings.any?
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/storage_check/gitlab_caller.rb b/lib/gitlab/storage_check/gitlab_caller.rb
new file mode 100644
index 00000000000..44952b68844
--- /dev/null
+++ b/lib/gitlab/storage_check/gitlab_caller.rb
@@ -0,0 +1,39 @@
+require 'excon'
+
+module Gitlab
+  module StorageCheck
+    class GitlabCaller
+      def initialize(options)
+        @options = options
+      end
+
+      def call!
+        Gitlab::StorageCheck::Response.new(get_response)
+      rescue Errno::ECONNREFUSED, Excon::Error
+        # Server not ready, treated as invalid response.
+        Gitlab::StorageCheck::Response.new(nil)
+      end
+
+      def get_response
+        scheme, *other_parts = URI.split(@options.target)
+        socket_path = if  scheme == 'unix'
+                        other_parts.compact.join
+                      end
+
+        connection = Excon.new(@options.target, socket: socket_path)
+        connection.post(path: Gitlab::StorageCheck::ENDPOINT,
+                        headers: headers)
+      end
+
+      def headers
+        @headers ||= begin
+                       headers = {}
+                       headers['Content-Type'] = headers['Accept'] = 'application/json'
+                       headers['TOKEN'] = @options.token if @options.token
+
+                       headers
+                     end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/storage_check/option_parser.rb b/lib/gitlab/storage_check/option_parser.rb
new file mode 100644
index 00000000000..66ed7906f97
--- /dev/null
+++ b/lib/gitlab/storage_check/option_parser.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module StorageCheck
+    class OptionParser
+      def self.parse!(args)
+        # Start out with some defaults
+        options = Gitlab::StorageCheck::Options.new(nil, nil, 1, false)
+
+        parser = ::OptionParser.new do |opts|
+          opts.banner = "Usage: bin/storage_check [options]"
+
+          opts.on('-t=string', '--target string', 'URL or socket to trigger storage check') do |value|
+            options.target = value
+          end
+
+          opts.on('-T=string', '--token string', 'Health token to use') { |value| options.token = value }
+
+          opts.on('-i=n', '--interval n', ::OptionParser::DecimalInteger, 'Seconds between checks') do |value|
+            options.interval = value
+          end
+
+          opts.on('-d', '--dryrun', "Output what will be performed, but don't start the process") do |value|
+            options.dryrun = value
+          end
+        end
+        parser.parse!(args)
+
+        unless options.target
+          raise ::OptionParser::InvalidArgument.new('Provide a URI to provide checks')
+        end
+
+        if URI.parse(options.target).scheme.nil?
+          raise ::OptionParser::InvalidArgument.new('Add the scheme to the target, `unix://`, `https://` or `http://` are supported')
+        end
+
+        options
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/storage_check/response.rb b/lib/gitlab/storage_check/response.rb
new file mode 100644
index 00000000000..326ab236e3e
--- /dev/null
+++ b/lib/gitlab/storage_check/response.rb
@@ -0,0 +1,77 @@
+require 'json'
+
+module Gitlab
+  module StorageCheck
+    class Response
+      attr_reader :http_response
+
+      def initialize(http_response)
+        @http_response = http_response
+      end
+
+      def valid?
+        @http_response && (200...299).cover?(@http_response.status) &&
+          @http_response.headers['Content-Type'].include?('application/json') &&
+          parsed_response
+      end
+
+      def check_interval
+        return nil unless parsed_response
+
+        parsed_response['check_interval']
+      end
+
+      def responsive_shards
+        divided_results[:responsive_shards]
+      end
+
+      def skipped_shards
+        divided_results[:skipped_shards]
+      end
+
+      def failing_shards
+        divided_results[:failing_shards]
+      end
+
+      private
+
+      def results
+        return [] unless parsed_response
+
+        parsed_response['results']
+      end
+
+      def divided_results
+        return @divided_results if @divided_results
+
+        @divided_results = {}
+        @divided_results[:responsive_shards] = []
+        @divided_results[:skipped_shards] = []
+        @divided_results[:failing_shards] = []
+
+        results.each do |info|
+          name = info['storage']
+
+          case info['success']
+          when true
+            @divided_results[:responsive_shards] << name
+          when false
+            @divided_results[:failing_shards] << name
+          else
+            @divided_results[:skipped_shards] << name
+          end
+        end
+
+        @divided_results
+      end
+
+      def parsed_response
+        return @parsed_response if defined?(@parsed_response)
+
+        @parsed_response = JSON.parse(@http_response.body)
+      rescue JSON::JSONError
+        @parsed_response = nil
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tcp_checker.rb b/lib/gitlab/tcp_checker.rb
new file mode 100644
index 00000000000..6e24e46d0ea
--- /dev/null
+++ b/lib/gitlab/tcp_checker.rb
@@ -0,0 +1,45 @@
+module Gitlab
+  class TcpChecker
+    attr_reader :remote_host, :remote_port, :local_host, :local_port, :error
+
+    def initialize(remote_host, remote_port, local_host = nil, local_port = nil)
+      @remote_host = remote_host
+      @remote_port = remote_port
+      @local_host = local_host
+      @local_port = local_port
+    end
+
+    def local
+      join_host_port(local_host, local_port)
+    end
+
+    def remote
+      join_host_port(remote_host, remote_port)
+    end
+
+    def check(timeout: 10)
+      Socket.tcp(
+        remote_host, remote_port,
+        local_host, local_port,
+        connect_timeout: timeout
+      ) do |sock|
+        @local_port, @local_host = Socket.unpack_sockaddr_in(sock.local_address)
+        @remote_port, @remote_host = Socket.unpack_sockaddr_in(sock.remote_address)
+      end
+
+      true
+    rescue => err
+      @error = err
+
+      false
+    end
+
+    private
+
+    def join_host_port(host, port)
+      host = "[#{host}]" if host.include?(':')
+
+      "#{host}:#{port}"
+    end
+  end
+end
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index abb3d3a02c3..b3baaf036d8 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -46,5 +46,22 @@ module Gitlab
     def random_string
       Random.rand(Float::MAX.to_i).to_s(36)
     end
+
+    # See: http://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby
+    # Cross-platform way of finding an executable in the $PATH.
+    #
+    #   which('ruby') #=> /usr/bin/ruby
+    def which(cmd, env = ENV)
+      exts = env['PATHEXT'] ? env['PATHEXT'].split(';') : ['']
+
+      env['PATH'].split(File::PATH_SEPARATOR).each do |path|
+        exts.each do |ext|
+          exe = File.join(path, "#{cmd}#{ext}")
+          return exe if File.executable?(exe) && !File.directory?(exe)
+        end
+      end
+
+      nil
+    end
   end
 end
diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb
index a2ac9285b56..fe091f4611b 100644
--- a/lib/gitlab/utils/strong_memoize.rb
+++ b/lib/gitlab/utils/strong_memoize.rb
@@ -11,6 +11,8 @@ module Gitlab
       #
       # We could write it like:
       #
+      #     include Gitlab::Utils::StrongMemoize
+      #
       #     def trigger_from_token
       #       strong_memoize(:trigger) do
       #         Ci::Trigger.find_by_token(params[:token].to_s)
@@ -18,14 +20,22 @@ module Gitlab
       #     end
       #
       def strong_memoize(name)
-        ivar_name = "@#{name}"
-
-        if instance_variable_defined?(ivar_name)
-          instance_variable_get(ivar_name)
+        if instance_variable_defined?(ivar(name))
+          instance_variable_get(ivar(name))
         else
-          instance_variable_set(ivar_name, yield)
+          instance_variable_set(ivar(name), yield)
         end
       end
+
+      def clear_memoization(name)
+        remove_instance_variable(ivar(name)) if instance_variable_defined?(ivar(name))
+      end
+
+      private
+
+      def ivar(name)
+        "@#{name}"
+      end
     end
   end
 end
diff --git a/lib/gitlab/view/presenter/factory.rb b/lib/gitlab/view/presenter/factory.rb
index d172d61e2c9..570f0723e39 100644
--- a/lib/gitlab/view/presenter/factory.rb
+++ b/lib/gitlab/view/presenter/factory.rb
@@ -16,7 +16,7 @@ module Gitlab
         attr_reader :subject, :attributes
 
         def presenter_class
-          "#{subject.class.name}Presenter".constantize
+          attributes.delete(:presenter_class) { "#{subject.class.name}Presenter".constantize }
         end
       end
     end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 864a9e04888..5ab6cd5a4ef 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -58,7 +58,7 @@ module Gitlab
       end
 
       def artifact_upload_ok
-        { TempPath: ArtifactUploader.artifacts_upload_path }
+        { TempPath: JobArtifactUploader.artifacts_upload_path }
       end
 
       def send_git_blob(repository, blob)