Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (168 commits)
  Update CHANGELOG.md for 10.0.1
  Remove Grit settings from default settings
  Fix duplicate key errors in PostDeployMigrateUserExternalMailData migration
  Workaround for #38259
  Workaround for n+1 in Projects::TreeController#show
  Removed old icons from project page
  Make branches page translatable
  fix typo in icons section
  Don't show it if there's no project.
  Update CHANGELOG.md for 10.0.0
  Inform user that current shared projects will remain shared
  Allow the git circuit breaker to correctly handle missing repository storages
  Reserve refs/replace cos `git-replace` is using it
  Resolve "Better SVG Usage in the Frontend"
  Replace the 'project/service.feature' spinach test with an rspec analog
  Replace the 'project/shortcuts.feature' spinach test with an rspec analog
  Removed two legacy config options
  Fix rendering double note issue.
  IssueNotes: Switch back to Write pane when note cancel or submit.
  Upgrade Nokogiri because of CVE-2017-9050
  ...

39 files changed, 685 insertions, 196 deletions

diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index 642c1140fcc..643c8e6fb8e 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -21,7 +21,10 @@ module API
       get ':id/repository/branches' do
         branches = ::Kaminari.paginate_array(user_project.repository.branches.sort_by(&:name))
 
-        present paginate(branches), with: Entities::RepoBranch, project: user_project
+        # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37442
+        Gitlab::GitalyClient.allow_n_plus_1_calls do
+          present paginate(branches), with: Entities::RepoBranch, project: user_project
+        end
       end
 
       resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 52c49e5caa9..71253f72533 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -244,7 +244,10 @@ module API
       end
 
       expose :merged do |repo_branch, options|
-        options[:project].repository.merged_to_root_ref?(repo_branch.name)
+        # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37442
+        Gitlab::GitalyClient.allow_n_plus_1_calls do
+          options[:project].repository.merged_to_root_ref?(repo_branch.name)
+        end
       end
 
       expose :protected do |repo_branch, options|
@@ -332,6 +335,7 @@ module API
     end
 
     class IssueBasic < ProjectEntity
+      expose :closed_at
       expose :labels do |issue, options|
         # Avoids an N+1 query since labels are preloaded
         issue.labels.map(&:title).sort
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 56d72d511da..8aa1e0216ee 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -2,7 +2,7 @@ module API
   class MergeRequests < Grape::API
     include PaginationParams
 
-    before { authenticate! }
+    before { authenticate_non_get! }
 
     helpers ::Gitlab::IssuableMetadata
 
@@ -55,6 +55,7 @@ module API
                          desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`'
       end
       get do
+        authenticate! unless params[:scope] == 'all'
         merge_requests = find_merge_requests
 
         options = { with: Entities::MergeRequestBasic,
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 7dc19788462..aab7a6c3f93 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -70,8 +70,11 @@ module API
         optional :import_url, type: String, desc: 'URL from which the project is imported'
       end
 
-      def present_projects(options = {})
-        projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
+      def load_projects
+        ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
+      end
+
+      def present_projects(projects, options = {})
         projects = reorder_projects(projects)
         projects = projects.with_statistics if params[:statistics]
         projects = projects.with_issues_enabled if params[:with_issues_enabled]
@@ -111,7 +114,7 @@ module API
 
         params[:user] = user
 
-        present_projects
+        present_projects load_projects
       end
     end
 
@@ -124,7 +127,7 @@ module API
         use :statistics_params
       end
       get do
-        present_projects
+        present_projects load_projects
       end
 
       desc 'Create new project' do
@@ -229,6 +232,18 @@ module API
         end
       end
 
+      desc 'List forks of this project' do
+        success Entities::Project
+      end
+      params do
+        use :collection_params
+      end
+      get ':id/forks' do
+        forks = ForkProjectsFinder.new(user_project, params: project_finder_params, current_user: current_user).execute
+
+        present_projects forks
+      end
+
       desc 'Update an existing project' do
         success Entities::Project
       end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index b9a573d3542..3cf3939994a 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -79,7 +79,7 @@ module Backup
             # - 1495527122_gitlab_backup.tar
             # - 1495527068_2017_05_23_gitlab_backup.tar
             # - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar
-            next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+.*)?)?_gitlab_backup\.tar$/
+            next unless file =~ /^(\d{10})(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+((-|\.)(pre|rc\d))?(-ee)?)?)?_gitlab_backup\.tar$/
 
             timestamp = $1.to_i
 
diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb
index a65bbe23958..e0a8ca653cb 100644
--- a/lib/banzai/reference_parser/issue_parser.rb
+++ b/lib/banzai/reference_parser/issue_parser.rb
@@ -34,7 +34,8 @@ module Banzai
                 { namespace: :owner },
                 { group: [:owners, :group_members] },
                 :invited_groups,
-                :project_members
+                :project_members,
+                :project_feature
               ]
             }
           ),
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 11ace83c15c..87aeb76b66a 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -2,7 +2,7 @@ module Gitlab
   module Auth
     MissingPersonalTokenError = Class.new(StandardError)
 
-    REGISTRY_SCOPES = Gitlab.config.registry.enabled ? [:read_registry].freeze : [].freeze
+    REGISTRY_SCOPES = [:read_registry].freeze
 
     # Scopes used for GitLab API access
     API_SCOPES = [:api, :read_user].freeze
@@ -13,11 +13,6 @@ module Gitlab
     # Default scopes for OAuth applications that don't define their own
     DEFAULT_SCOPES = [:api].freeze
 
-    AVAILABLE_SCOPES = (API_SCOPES + REGISTRY_SCOPES).freeze
-
-    # Other available scopes
-    OPTIONAL_SCOPES = (AVAILABLE_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze
-
     class << self
       include Gitlab::CurrentSettings
 
@@ -132,7 +127,7 @@ module Gitlab
 
         token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password)
 
-        if token && valid_scoped_token?(token, AVAILABLE_SCOPES)
+        if token && valid_scoped_token?(token, available_scopes)
           Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes))
         end
       end
@@ -230,6 +225,21 @@ module Gitlab
       def read_user_scope_authentication_abilities
         []
       end
+
+      def available_scopes
+        API_SCOPES + registry_scopes
+      end
+
+      # Other available scopes
+      def optional_scopes
+        available_scopes + OPENID_SCOPES - DEFAULT_SCOPES
+      end
+
+      def registry_scopes
+        return [] unless Gitlab.config.registry.enabled
+
+        REGISTRY_SCOPES
+      end
     end
   end
 end
diff --git a/lib/gitlab/ci/build/policy.rb b/lib/gitlab/ci/build/policy.rb
new file mode 100644
index 00000000000..d10cc7802d4
--- /dev/null
+++ b/lib/gitlab/ci/build/policy.rb
@@ -0,0 +1,15 @@
+module Gitlab
+  module Ci
+    module Build
+      module Policy
+        def self.fabricate(specs)
+          specifications = specs.to_h.map do |spec, value|
+            self.const_get(spec.to_s.camelize).new(value)
+          end
+
+          specifications.compact
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb
new file mode 100644
index 00000000000..b20d374288f
--- /dev/null
+++ b/lib/gitlab/ci/build/policy/kubernetes.rb
@@ -0,0 +1,19 @@
+module Gitlab
+  module Ci
+    module Build
+      module Policy
+        class Kubernetes < Policy::Specification
+          def initialize(spec)
+            unless spec.to_sym == :active
+              raise UnknownPolicyError
+            end
+          end
+
+          def satisfied_by?(pipeline)
+            pipeline.has_kubernetes_active?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb
new file mode 100644
index 00000000000..eadc0948d2f
--- /dev/null
+++ b/lib/gitlab/ci/build/policy/refs.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module Ci
+    module Build
+      module Policy
+        class Refs < Policy::Specification
+          def initialize(refs)
+            @patterns = Array(refs)
+          end
+
+          def satisfied_by?(pipeline)
+            @patterns.any? do |pattern|
+              pattern, path = pattern.split('@', 2)
+
+              matches_path?(path, pipeline) &&
+                matches_pattern?(pattern, pipeline)
+            end
+          end
+
+          private
+
+          def matches_path?(path, pipeline)
+            return true unless path
+
+            pipeline.project_full_path == path
+          end
+
+          def matches_pattern?(pattern, pipeline)
+            return true if pipeline.tag? && pattern == 'tags'
+            return true if pipeline.branch? && pattern == 'branches'
+            return true if pipeline.source == pattern
+            return true if pipeline.source&.pluralize == pattern
+
+            if pattern.first == "/" && pattern.last == "/"
+              Regexp.new(pattern[1...-1]) =~ pipeline.ref
+            else
+              pattern == pipeline.ref
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb
new file mode 100644
index 00000000000..c317291f29d
--- /dev/null
+++ b/lib/gitlab/ci/build/policy/specification.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module Ci
+    module Build
+      module Policy
+        ##
+        # Abstract class that defines an interface of job policy
+        # specification.
+        #
+        # Used for job's only/except policy configuration.
+        #
+        class Specification
+          UnknownPolicyError = Class.new(StandardError)
+
+          def initialize(spec)
+            @spec = spec
+          end
+
+          def satisfied_by?(pipeline)
+            raise NotImplementedError
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 7582964b24e..0bd78b03448 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -5,12 +5,11 @@ module Gitlab
 
       include Gitlab::Ci::Config::Entry::LegacyValidationHelpers
 
-      attr_reader :path, :cache, :stages, :jobs
+      attr_reader :cache, :stages, :jobs
 
-      def initialize(config, path = nil)
+      def initialize(config)
         @ci_config = Gitlab::Ci::Config.new(config)
         @config = @ci_config.to_hash
-        @path = path
 
         unless @ci_config.valid?
           raise ValidationError, @ci_config.errors.first
@@ -21,28 +20,12 @@ module Gitlab
         raise ValidationError, e.message
       end
 
-      def builds_for_stage_and_ref(stage, ref, tag = false, source = nil)
-        jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _|
-          build_attributes(name)
-        end
-      end
-
       def builds
         @jobs.map do |name, _|
           build_attributes(name)
         end
       end
 
-      def stage_seeds(pipeline)
-        seeds = @stages.uniq.map do |stage|
-          builds = pipeline_stage_builds(stage, pipeline)
-
-          Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any?
-        end
-
-        seeds.compact
-      end
-
       def build_attributes(name)
         job = @jobs[name.to_sym] || {}
 
@@ -70,6 +53,32 @@ module Gitlab
           }.compact }
       end
 
+      def pipeline_stage_builds(stage, pipeline)
+        selected_jobs = @jobs.select do |_, job|
+          next unless job[:stage] == stage
+
+          only_specs = Gitlab::Ci::Build::Policy
+            .fabricate(job.fetch(:only, {}))
+          except_specs = Gitlab::Ci::Build::Policy
+            .fabricate(job.fetch(:except, {}))
+
+          only_specs.all? { |spec| spec.satisfied_by?(pipeline) } &&
+            except_specs.none? { |spec| spec.satisfied_by?(pipeline) }
+        end
+
+        selected_jobs.map { |_, job| build_attributes(job[:name]) }
+      end
+
+      def stage_seeds(pipeline)
+        seeds = @stages.uniq.map do |stage|
+          builds = pipeline_stage_builds(stage, pipeline)
+
+          Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any?
+        end
+
+        seeds.compact
+      end
+
       def self.validation_message(content)
         return 'Please provide content of .gitlab-ci.yml' if content.blank?
 
@@ -83,34 +92,6 @@ module Gitlab
 
       private
 
-      def pipeline_stage_builds(stage, pipeline)
-        builds = builds_for_stage_and_ref(
-          stage, pipeline.ref, pipeline.tag?, pipeline.source)
-
-        builds.select do |build|
-          job = @jobs[build.fetch(:name).to_sym]
-          has_kubernetes = pipeline.has_kubernetes_active?
-          only_kubernetes = job.dig(:only, :kubernetes)
-          except_kubernetes = job.dig(:except, :kubernetes)
-
-          [!only_kubernetes && !except_kubernetes,
-           only_kubernetes && has_kubernetes,
-           except_kubernetes && !has_kubernetes].any?
-        end
-      end
-
-      def jobs_for_ref(ref, tag = false, source = nil)
-        @jobs.select do |_, job|
-          process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source)
-        end
-      end
-
-      def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil)
-        jobs_for_ref(ref, tag, source).select do |_, job|
-          job[:stage] == stage
-        end
-      end
-
       def initial_parsing
         ##
         # Global config
@@ -203,51 +184,6 @@ module Gitlab
           raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined"
         end
       end
-
-      def process?(only_params, except_params, ref, tag, source)
-        if only_params.present?
-          return false unless matching?(only_params, ref, tag, source)
-        end
-
-        if except_params.present?
-          return false if matching?(except_params, ref, tag, source)
-        end
-
-        true
-      end
-
-      def matching?(patterns, ref, tag, source)
-        patterns.any? do |pattern|
-          pattern, path = pattern.split('@', 2)
-          matches_path?(path) && matches_pattern?(pattern, ref, tag, source)
-        end
-      end
-
-      def matches_path?(path)
-        return true unless path
-
-        path == self.path
-      end
-
-      def matches_pattern?(pattern, ref, tag, source)
-        return true if tag && pattern == 'tags'
-        return true if !tag && pattern == 'branches'
-        return true if source_to_pattern(source) == pattern
-
-        if pattern.first == "/" && pattern.last == "/"
-          Regexp.new(pattern[1...-1]) =~ ref
-        else
-          pattern == ref
-        end
-      end
-
-      def source_to_pattern(source)
-        if %w[api external web].include?(source)
-          source
-        else
-          source&.pluralize
-        end
-      end
     end
   end
 end
diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb
index 58f86abc5c4..243c1f1394d 100644
--- a/lib/gitlab/closing_issue_extractor.rb
+++ b/lib/gitlab/closing_issue_extractor.rb
@@ -1,7 +1,7 @@
 module Gitlab
   class ClosingIssueExtractor
     ISSUE_CLOSING_REGEX = begin
-      link_pattern = URI.regexp(%w(http https))
+      link_pattern = Banzai::Filter::AutolinkFilter::LINK_PATTERN
 
       pattern = Gitlab.config.gitlab.issue_closing_pattern
       pattern = pattern.sub('%{issue_ref}', "(?:(?:#{link_pattern})|(?:#{Issue.reference_pattern}))")
diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb
index 4ab5b3455a5..31a46a738c3 100644
--- a/lib/gitlab/data_builder/push.rb
+++ b/lib/gitlab/data_builder/push.rb
@@ -64,8 +64,11 @@ module Gitlab
 
         # For performance purposes maximum 20 latest commits
         # will be passed as post receive hook data.
-        commit_attrs = commits_limited.map do |commit|
-          commit.hook_attrs(with_changed_files: true)
+        # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/38259
+        commit_attrs = Gitlab::GitalyClient.allow_n_plus_1_calls do
+          commits_limited.map do |commit|
+            commit.hook_attrs(with_changed_files: true)
+          end
         end
 
         type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push'
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index 1dabd4ebdd0..fcac85ff892 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -5,7 +5,7 @@ module Gitlab
 
       delegate :new_file?, :deleted_file?, :renamed_file?,
         :old_path, :new_path, :a_mode, :b_mode, :mode_changed?,
-        :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, to: :diff, prefix: false
+        :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, :has_binary_notice?, to: :diff, prefix: false
 
       # Finding a viewer for a diff file happens based only on extension and whether the
       # diff file blobs are binary or text, which means 1 diff file should only be matched by 1 viewer,
@@ -166,7 +166,7 @@ module Gitlab
       end
 
       def binary?
-        old_blob&.binary? || new_blob&.binary?
+        has_binary_notice? || old_blob&.binary? || new_blob&.binary?
       end
 
       def text?
diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb
index a6007ebf531..88ae65cb468 100644
--- a/lib/gitlab/diff/file_collection/base.rb
+++ b/lib/gitlab/diff/file_collection/base.rb
@@ -22,7 +22,10 @@ module Gitlab
         end
 
         def diff_files
-          @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) }
+          # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37445
+          Gitlab::GitalyClient.allow_n_plus_1_calls do
+            @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) }
+          end
         end
 
         def diff_file_with_old_path(old_path)
diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb
index abd401224d8..c5a8ea12245 100644
--- a/lib/gitlab/ee_compat_check.rb
+++ b/lib/gitlab/ee_compat_check.rb
@@ -284,13 +284,18 @@ module Gitlab
         EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch
         was found in the EE repository.
 
+        If you're a community contributor, don't worry, someone from
+        GitLab Inc. will take care of this, and you don't have to do anything.
+        If you're willing to help, and are ok to contribute to EE as well,
+        you're welcome to help. You could follow the instructions below.
+
         #{conflicting_files_msg}
 
         We advise you to create a `#{ee_branch_prefix}` or `#{ee_branch_suffix}`
         branch that includes changes from `#{ce_branch}` but also specific changes
         than can be applied cleanly to EE/master. In some cases, the conflicts
         are trivial and you can ignore the warning from this job. As always,
-        use your best judgment!
+        use your best judgement!
 
         There are different ways to create such branch:
 
diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb
index b984492d369..455814a9159 100644
--- a/lib/gitlab/gfm/reference_rewriter.rb
+++ b/lib/gitlab/gfm/reference_rewriter.rb
@@ -29,6 +29,8 @@ module Gitlab
     #  http://gitlab.com/some/link/#1234, and code `puts #1234`'
     #
     class ReferenceRewriter
+      RewriteError = Class.new(StandardError)
+
       def initialize(text, source_project, current_user)
         @text = text
         @source_project = source_project
@@ -61,6 +63,10 @@ module Gitlab
         cross_reference = build_cross_reference(referable, target_project)
         return reference if reference == cross_reference
 
+        if cross_reference.nil?
+          raise RewriteError, "Unspecified reference detected for #{referable.class.name}"
+        end
+
         new_text = before + cross_reference + after
         substitution_valid?(new_text) ? cross_reference : reference
       end
diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index b4b6326cfdd..c78fe63f9b5 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -57,6 +57,15 @@ module Gitlab
       def version
         Gitlab::VersionInfo.parse(Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --version)).first)
       end
+
+      def check_namespace!(*objects)
+        expected_namespace = self.name + '::'
+        objects.each do |object|
+          unless object.class.name.start_with?(expected_namespace)
+            raise ArgumentError, "expected object in #{expected_namespace}, got #{object}"
+          end
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index 8d96826f6ee..a4336facee5 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -32,6 +32,8 @@ module Gitlab
             else
               blob = repository.lookup(sha)
 
+              next unless blob.is_a?(Rugged::Blob)
+
               new(
                 id: blob.oid,
                 size: blob.size,
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 1f370686186..1957c254c28 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -413,6 +413,10 @@ module Gitlab
                            end
       end
 
+      def merge_commit?
+        parent_ids.size > 1
+      end
+
       private
 
       def init_from_hash(hash)
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index a23c8cf0dd1..096301d300f 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -206,6 +206,10 @@ module Gitlab
         Diff.binary_message(@old_path, @new_path)
       end
 
+      def has_binary_notice?
+        @diff.start_with?('Binary')
+      end
+
       private
 
       def init_from_rugged(rugged)
diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb
index cc35d77c6e4..208e4bbaf60 100644
--- a/lib/gitlab/git/hook.rb
+++ b/lib/gitlab/git/hook.rb
@@ -83,13 +83,14 @@ module Gitlab
       def call_update_hook(gl_id, oldrev, newrev, ref)
         Dir.chdir(repo_path) do
           stdout, stderr, status = Open3.capture3({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev)
-          [status.success?, stderr.presence || stdout]
+          [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe]
         end
       end
 
       def retrieve_error_message(stderr, stdout)
-        err_message = stderr.gets
-        err_message.blank? ? stdout.gets : err_message
+        err_message = stderr.read
+        err_message = err_message.blank? ? stdout.read : err_message
+        err_message.gsub(/\R/, "<br>").html_safe
       end
     end
   end
diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb
index dcdec818f5e..786e2e7e8dc 100644
--- a/lib/gitlab/git/operation_service.rb
+++ b/lib/gitlab/git/operation_service.rb
@@ -1,6 +1,8 @@
 module Gitlab
   module Git
     class OperationService
+      include Gitlab::Git::Popen
+
       WithBranchResult = Struct.new(:newrev, :repo_created, :branch_created) do
         alias_method :repo_created?, :repo_created
         alias_method :branch_created?, :branch_created
@@ -15,9 +17,7 @@ module Gitlab
         end
 
         # Refactoring aid
-        unless new_repository.is_a?(Gitlab::Git::Repository)
-          raise "expected a Gitlab::Git::Repository, got #{new_repository}"
-        end
+        Gitlab::Git.check_namespace!(new_repository)
 
         @repository = new_repository
       end
@@ -152,7 +152,7 @@ module Gitlab
         # (and have!) accidentally reset the ref to an earlier state, clobbering
         # commits. See also https://github.com/libgit2/libgit2/issues/1534.
         command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z]
-        _, status = Gitlab::Popen.popen(
+        _, status = popen(
           command,
           repository.path) do |stdin|
           stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00")
diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb
index 10c15b316f5..054d45895a5 100644
--- a/lib/gitlab/git/popen.rb
+++ b/lib/gitlab/git/popen.rb
@@ -5,17 +5,21 @@ require 'open3'
 module Gitlab
   module Git
     module Popen
-      def popen(cmd, path)
+      def popen(cmd, path, vars = {})
         unless cmd.is_a?(Array)
           raise "System commands must be given as an array of strings"
         end
 
-        vars = { "PWD" => path }
+        path ||= Dir.pwd
+        vars['PWD'] = path
         options = { chdir: path }
 
         cmd_output = ""
         cmd_status = 0
         Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|
+          yield(stdin) if block_given?
+          stdin.close
+
           cmd_output << stdout.read
           cmd_output << stderr.read
           cmd_status = wait_thr.value.exitstatus
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index c499ff101b5..10ba29acbd1 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -19,6 +19,7 @@ module Gitlab
       InvalidRef = Class.new(StandardError)
       GitError = Class.new(StandardError)
       DeleteBranchError = Class.new(StandardError)
+      CreateTreeError = Class.new(StandardError)
 
       class << self
         # Unlike `new`, `create` takes the storage path, not the storage name
@@ -474,7 +475,15 @@ module Gitlab
       # diff options.  The +options+ hash can also include :break_rewrites to
       # split larger rewrites into delete/add pairs.
       def diff(from, to, options = {}, *paths)
-        Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options)
+        iterator = gitaly_migrate(:diff_between) do |is_enabled|
+          if is_enabled
+            gitaly_commit_client.diff(from, to, options.merge(paths: paths))
+          else
+            diff_patches(from, to, options, *paths)
+          end
+        end
+
+        Gitlab::Git::DiffCollection.new(iterator, options)
       end
 
       # Returns a RefName for a given SHA
@@ -489,7 +498,7 @@ module Gitlab
 
             # Not found -> ["", 0]
             # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0]
-            Gitlab::Popen.popen(args, @path).first.split.last
+            popen(args, @path).first.split.last
           end
         end
       end
@@ -684,6 +693,88 @@ module Gitlab
         nil
       end
 
+      def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+        OperationService.new(user, self).with_branch(
+          branch_name,
+          start_branch_name: start_branch_name,
+          start_repository: start_repository
+        ) do |start_commit|
+
+          Gitlab::Git.check_namespace!(commit, start_repository)
+
+          revert_tree_id = check_revert_content(commit, start_commit.sha)
+          raise CreateTreeError unless revert_tree_id
+
+          committer = user_to_committer(user)
+
+          create_commit(message: message,
+                        author: committer,
+                        committer: committer,
+                        tree: revert_tree_id,
+                        parents: [start_commit.sha])
+        end
+      end
+
+      def check_revert_content(target_commit, source_sha)
+        args = [target_commit.sha, source_sha]
+        args << { mainline: 1 } if target_commit.merge_commit?
+
+        revert_index = rugged.revert_commit(*args)
+        return false if revert_index.conflicts?
+
+        tree_id = revert_index.write_tree(rugged)
+        return false unless diff_exists?(source_sha, tree_id)
+
+        tree_id
+      end
+
+      def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+        OperationService.new(user, self).with_branch(
+          branch_name,
+          start_branch_name: start_branch_name,
+          start_repository: start_repository
+        ) do |start_commit|
+
+          Gitlab::Git.check_namespace!(commit, start_repository)
+
+          cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha)
+          raise CreateTreeError unless cherry_pick_tree_id
+
+          committer = user_to_committer(user)
+
+          create_commit(message: message,
+                        author: {
+                            email: commit.author_email,
+                            name: commit.author_name,
+                            time: commit.authored_date
+                        },
+                        committer: committer,
+                        tree: cherry_pick_tree_id,
+                        parents: [start_commit.sha])
+        end
+      end
+
+      def check_cherry_pick_content(target_commit, source_sha)
+        args = [target_commit.sha, source_sha]
+        args << 1 if target_commit.merge_commit?
+
+        cherry_pick_index = rugged.cherrypick_commit(*args)
+        return false if cherry_pick_index.conflicts?
+
+        tree_id = cherry_pick_index.write_tree(rugged)
+        return false unless diff_exists?(source_sha, tree_id)
+
+        tree_id
+      end
+
+      def diff_exists?(sha1, sha2)
+        rugged.diff(sha1, sha2).size > 0
+      end
+
+      def user_to_committer(user)
+        Gitlab::Git.committer_hash(email: user.email, name: user.name)
+      end
+
       def create_commit(params = {})
         params[:message].delete!("\r")
 
@@ -709,9 +800,7 @@ module Gitlab
         end
 
         command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z]
-        message, status = Gitlab::Popen.popen(
-          command,
-          path) do |stdin|
+        message, status = popen(command, path) do |stdin|
           stdin.write(instructions.join)
         end
 
@@ -835,7 +924,7 @@ module Gitlab
       end
 
       def with_repo_branch_commit(start_repository, start_branch_name)
-        raise "expected Gitlab::Git::Repository, got #{start_repository}" unless start_repository.is_a?(Gitlab::Git::Repository)
+        Gitlab::Git.check_namespace!(start_repository)
 
         return yield nil if start_repository.empty_repo?
 
@@ -950,8 +1039,8 @@ module Gitlab
         @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self)
       end
 
-      def gitaly_migrate(method, &block)
-        Gitlab::GitalyClient.migrate(method, &block)
+      def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
+        Gitlab::GitalyClient.migrate(method, status: status, &block)
       rescue GRPC::NotFound => e
         raise NoRepository.new(e)
       rescue GRPC::BadStatus => e
@@ -962,14 +1051,17 @@ module Gitlab
 
       # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'.
       def branches_filter(filter: nil, sort_by: nil)
-        branches = rugged.branches.each(filter).map do |rugged_ref|
-          begin
-            target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
-            Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
-          rescue Rugged::ReferenceError
-            # Omit invalid branch
-          end
-        end.compact
+        # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464
+        branches = Gitlab::GitalyClient.allow_n_plus_1_calls do
+          rugged.branches.each(filter).map do |rugged_ref|
+            begin
+              target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
+              Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
+            rescue Rugged::ReferenceError
+              # Omit invalid branch
+            end
+          end.compact
+        end
 
         sort_branches(branches, sort_by)
       end
diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb
index 2b5785a1f08..e0943d3a3eb 100644
--- a/lib/gitlab/git/rev_list.rb
+++ b/lib/gitlab/git/rev_list.rb
@@ -3,6 +3,8 @@
 module Gitlab
   module Git
     class RevList
+      include Gitlab::Git::Popen
+
       attr_reader :oldrev, :newrev, :path_to_repo
 
       def initialize(path_to_repo:, newrev:, oldrev: nil)
@@ -26,7 +28,7 @@ module Gitlab
       private
 
       def execute(args)
-        output, status = Gitlab::Popen.popen(args, nil, Gitlab::Git::Env.all.stringify_keys)
+        output, status = popen(args, nil, Gitlab::Git::Env.all.stringify_keys)
 
         unless status.zero?
           raise "Got a non-zero exit code while calling out `#{args.join(' ')}`."
diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb
index e28be4b8a38..08e6c29abad 100644
--- a/lib/gitlab/git/storage.rb
+++ b/lib/gitlab/git/storage.rb
@@ -11,6 +11,7 @@ module Gitlab
       end
 
       CircuitOpen = Class.new(Inaccessible)
+      Misconfiguration = Class.new(Inaccessible)
 
       REDIS_KEY_PREFIX = 'storage_accessible:'.freeze
 
diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb
index 9ea9367d4b7..1eaa2d83fb6 100644
--- a/lib/gitlab/git/storage/circuit_breaker.rb
+++ b/lib/gitlab/git/storage/circuit_breaker.rb
@@ -28,14 +28,26 @@ module Gitlab
         def self.for_storage(storage)
           cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do
             Hash.new do |hash, storage_name|
-              hash[storage_name] = new(storage_name)
+              hash[storage_name] = build(storage_name)
             end
           end
 
           cached_circuitbreakers[storage]
         end
 
-        def initialize(storage, hostname = Gitlab::Environment.hostname)
+        def self.build(storage, hostname = Gitlab::Environment.hostname)
+          config = Gitlab.config.repositories.storages[storage]
+
+          if !config.present?
+            NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured"))
+          elsif !config['path'].present?
+            NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured"))
+          else
+            new(storage, hostname)
+          end
+        end
+
+        def initialize(storage, hostname)
           @storage = storage
           @hostname = hostname
 
@@ -64,6 +76,10 @@ module Gitlab
           recent_failure || too_many_failures
         end
 
+        def failure_info
+          @failure_info ||= get_failure_info
+        end
+
         # Memoizing the `storage_available` call means we only do it once per
         # request when the storage is available.
         #
@@ -121,10 +137,12 @@ module Gitlab
           end
         end
 
-        def failure_info
-          @failure_info ||= get_failure_info
+        def cache_key
+          @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
         end
 
+        private
+
         def get_failure_info
           last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis|
             redis.hmget(cache_key, :last_failure, :failure_count)
@@ -134,10 +152,6 @@ module Gitlab
 
           FailureInfo.new(last_failure, failure_count.to_i)
         end
-
-        def cache_key
-          @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
-        end
       end
     end
   end
diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb
index 2d723147f4f..1564e94b7f7 100644
--- a/lib/gitlab/git/storage/health.rb
+++ b/lib/gitlab/git/storage/health.rb
@@ -78,7 +78,7 @@ module Gitlab
 
         def failing_circuit_breakers
           @failing_circuit_breakers ||= failing_on_hosts.map do |hostname|
-            CircuitBreaker.new(storage_name, hostname)
+            CircuitBreaker.build(storage_name, hostname)
           end
         end
 
diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb
new file mode 100644
index 00000000000..297c043d054
--- /dev/null
+++ b/lib/gitlab/git/storage/null_circuit_breaker.rb
@@ -0,0 +1,47 @@
+module Gitlab
+  module Git
+    module Storage
+      class NullCircuitBreaker
+        # These will have actual values
+        attr_reader :storage,
+                    :hostname
+
+        # These will always have nil values
+        attr_reader :storage_path,
+                    :failure_wait_time,
+                    :failure_reset_time,
+                    :storage_timeout
+
+        def initialize(storage, hostname, error: nil)
+          @storage = storage
+          @hostname = hostname
+          @error = error
+        end
+
+        def perform
+          @error ? raise(@error) : yield
+        end
+
+        def circuit_broken?
+          !!@error
+        end
+
+        def failure_count_threshold
+          1
+        end
+
+        def last_failure
+          circuit_broken? ? Time.now : nil
+        end
+
+        def failure_count
+          circuit_broken? ? 1 : 0
+        end
+
+        def failure_info
+          Gitlab::Git::Storage::CircuitBreaker::FailureInfo.new(last_failure, failure_count)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index a3dc2cd0b60..cbd9ff406de 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -10,7 +10,24 @@ module Gitlab
       OPT_OUT = 3
     end
 
+    class TooManyInvocationsError < StandardError
+      attr_reader :call_site, :invocation_count, :max_call_stack
+
+      def initialize(call_site, invocation_count, max_call_stack, most_invoked_stack)
+        @call_site = call_site
+        @invocation_count = invocation_count
+        @max_call_stack = max_call_stack
+        stacks = most_invoked_stack.join('\n') if most_invoked_stack
+
+        msg = "GitalyClient##{call_site} called #{invocation_count} times from single request. Potential n+1?"
+        msg << "\nThe following call site called into Gitaly #{max_call_stack} times:\n#{stacks}\n" if stacks
+
+        super(msg)
+      end
+    end
+
     SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze
+    MAXIMUM_GITALY_CALLS = 30
 
     MUTEX = Mutex.new
     private_constant :MUTEX
@@ -53,6 +70,8 @@ module Gitlab
     # All Gitaly RPC call sites should use GitalyClient.call. This method
     # makes sure that per-request authentication headers are set.
     def self.call(storage, service, rpc, request)
+      enforce_gitaly_request_limits(:call)
+
       metadata = request_metadata(storage)
       metadata = yield(metadata) if block_given?
       stub(service, storage).__send__(rpc, request, metadata) # rubocop:disable GitlabSecurity/PublicSend
@@ -107,12 +126,100 @@ module Gitlab
     private_class_method :opt_into_all_features?
 
     def self.migrate(feature, status: MigrationStatus::OPT_IN)
+      # Enforce limits at both the `migrate` and `call` sites to ensure that
+      # problems are not hidden by a feature being disabled
+      enforce_gitaly_request_limits(:migrate)
+
       is_enabled  = feature_enabled?(feature, status: status)
       metric_name = feature.to_s
       metric_name += "_gitaly" if is_enabled
 
       Gitlab::Metrics.measure(metric_name) do
-        yield is_enabled
+        # Some migrate calls wrap other migrate calls
+        allow_n_plus_1_calls do
+          yield is_enabled
+        end
+      end
+    end
+
+    # Ensures that Gitaly is not being abuse through n+1 misuse etc
+    def self.enforce_gitaly_request_limits(call_site)
+      # Only count limits in request-response environments (not sidekiq for example)
+      return unless RequestStore.active?
+
+      # This is this actual number of times this call was made. Used for information purposes only
+      actual_call_count = increment_call_count("gitaly_#{call_site}_actual")
+
+      # Do no enforce limits in production
+      return if Rails.env.production?
+
+      # Check if this call is nested within a allow_n_plus_1_calls
+      # block and skip check if it is
+      return if get_call_count(:gitaly_call_count_exception_block_depth) > 0
+
+      # This is the count of calls outside of a `allow_n_plus_1_calls` block
+      # It is used for enforcement but not statistics
+      permitted_call_count = increment_call_count("gitaly_#{call_site}_permitted")
+
+      count_stack
+
+      return if permitted_call_count <= MAXIMUM_GITALY_CALLS
+
+      raise TooManyInvocationsError.new(call_site, actual_call_count, max_call_count, max_stacks)
+    end
+
+    def self.allow_n_plus_1_calls
+      return yield unless RequestStore.active?
+
+      begin
+        increment_call_count(:gitaly_call_count_exception_block_depth)
+        yield
+      ensure
+        decrement_call_count(:gitaly_call_count_exception_block_depth)
+      end
+    end
+
+    def self.get_call_count(key)
+      RequestStore.store[key] || 0
+    end
+    private_class_method :get_call_count
+
+    def self.increment_call_count(key)
+      RequestStore.store[key] ||= 0
+      RequestStore.store[key] += 1
+    end
+    private_class_method :increment_call_count
+
+    def self.decrement_call_count(key)
+      RequestStore.store[key] -= 1
+    end
+    private_class_method :decrement_call_count
+
+    # Returns an estimate of the number of Gitaly calls made for this
+    # request
+    def self.get_request_count
+      return 0 unless RequestStore.active?
+
+      gitaly_migrate_count = get_call_count("gitaly_migrate_actual")
+      gitaly_call_count = get_call_count("gitaly_call_actual")
+
+      # Using the maximum of migrate and call_count will provide an
+      # indicator of how many Gitaly calls will be made, even
+      # before a feature is enabled. This provides us with a single
+      # metric, but not an exact number, but this tradeoff is acceptable
+      if gitaly_migrate_count > gitaly_call_count
+        gitaly_migrate_count
+      else
+        gitaly_call_count
+      end
+    end
+
+    def self.reset_counts
+      return unless RequestStore.active?
+
+      %w[migrate call].each do |call_site|
+        RequestStore.store["gitaly_#{call_site}_actual"] = 0
+        RequestStore.store["gitaly_#{call_site}_permitted"] = 0
       end
     end
 
@@ -124,5 +231,43 @@ module Gitlab
     def self.encode(s)
       s.dup.force_encoding(Encoding::ASCII_8BIT)
     end
+
+    # Count a stack. Used for n+1 detection
+    def self.count_stack
+      return unless RequestStore.active?
+
+      stack_string = caller.drop(1).join("\n")
+
+      RequestStore.store[:stack_counter] ||= Hash.new
+
+      count = RequestStore.store[:stack_counter][stack_string] || 0
+      RequestStore.store[:stack_counter][stack_string] = count + 1
+    end
+    private_class_method :count_stack
+
+    # Returns a count for the stack which called Gitaly the most times. Used for n+1 detection
+    def self.max_call_count
+      return 0 unless RequestStore.active?
+
+      stack_counter = RequestStore.store[:stack_counter]
+      return 0 unless stack_counter
+
+      stack_counter.values.max
+    end
+    private_class_method :max_call_count
+
+    # Returns the stacks that calls Gitaly the most times. Used for n+1 detection
+    def self.max_stacks
+      return nil unless RequestStore.active?
+
+      stack_counter = RequestStore.store[:stack_counter]
+      return nil unless stack_counter
+
+      max = max_call_count
+      return nil if max.zero?
+
+      stack_counter.select { |_, v| v == max }.keys
+    end
+    private_class_method :max_stacks
   end
 end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 1ba1a7830a4..cf3a3554552 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -32,20 +32,38 @@ module Gitlab
         GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request).value
       end
 
+      def diff(from, to, options = {})
+        from_id = case from
+                  when NilClass
+                    EMPTY_TREE_ID
+                  when Rugged::Commit
+                    from.oid
+                  else
+                    from
+                  end
+
+        to_id = case to
+                when NilClass
+                  EMPTY_TREE_ID
+                when Rugged::Commit
+                  to.oid
+                else
+                  to
+                end
+
+        request_params = diff_between_commits_request_params(from_id, to_id, options)
+
+        call_commit_diff(request_params, options)
+      end
+
       def diff_from_parent(commit, options = {})
-        request_params = commit_diff_request_params(commit, options)
-        request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false)
-        request_params[:enforce_limits] = options.fetch(:limits, true)
-        request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true)
-        request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h)
+        request_params = diff_from_parent_request_params(commit, options)
 
-        request = Gitaly::CommitDiffRequest.new(request_params)
-        response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request)
-        GitalyClient::DiffStitcher.new(response)
+        call_commit_diff(request_params, options)
       end
 
       def commit_deltas(commit)
-        request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit))
+        request = Gitaly::CommitDeltaRequest.new(diff_from_parent_request_params(commit))
         response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request)
 
         response.flat_map { |msg| msg.deltas }
@@ -214,14 +232,29 @@ module Gitlab
 
       private
 
-      def commit_diff_request_params(commit, options = {})
+      def call_commit_diff(request_params, options = {})
+        request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false)
+        request_params[:enforce_limits] = options.fetch(:limits, true)
+        request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true)
+        request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h)
+
+        request = Gitaly::CommitDiffRequest.new(request_params)
+        response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request)
+        GitalyClient::DiffStitcher.new(response)
+      end
+
+      def diff_from_parent_request_params(commit, options = {})
         parent_id = commit.parent_ids.first || EMPTY_TREE_ID
 
+        diff_between_commits_request_params(parent_id, commit.id, options)
+      end
+
+      def diff_between_commits_request_params(from_id, to_id, options)
         {
           repository: @gitaly_repo,
-          left_commit_id: parent_id,
-          right_commit_id: commit.id,
-          paths: options.fetch(:paths, [])
+          left_commit_id: from_id,
+          right_commit_id: to_id,
+          paths: options.fetch(:paths, []).map { |path| GitalyClient.encode(path) }
         }
       end
 
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 9bcc579278f..3a666c2268b 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -20,6 +20,7 @@ module Gitlab
       gon.gitlab_url             = Gitlab.config.gitlab.url
       gon.revision               = Gitlab::REVISION
       gon.gitlab_logo            = ActionController::Base.helpers.asset_path('gitlab_logo.png')
+      gon.sprite_icons           = ActionController::Base.helpers.asset_path('icons.svg')
 
       if current_user
         gon.current_user_id = current_user.id
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb
index eef97f54962..afaa59b1018 100644
--- a/lib/gitlab/health_checks/fs_shards_check.rb
+++ b/lib/gitlab/health_checks/fs_shards_check.rb
@@ -58,11 +58,11 @@ module Gitlab
         end
 
         def repository_storages
-          @repository_storage ||= Gitlab::CurrentSettings.current_application_settings.repository_storages
+          storages_paths.keys
         end
 
         def storages_paths
-          @storage_paths ||= Gitlab.config.repositories.storages
+          Gitlab.config.repositories.storages
         end
 
         def exec_with_timeout(cmd_args, *args, &block)
@@ -125,7 +125,7 @@ module Gitlab
         end
 
         def storage_circuitbreaker_test(storage_name)
-          Gitlab::Git::Storage::CircuitBreaker.new(storage_name).perform { "OK" }
+          Gitlab::Git::Storage::CircuitBreaker.build(storage_name).perform { "OK" }
         rescue Gitlab::Git::Storage::Inaccessible
           nil
         end
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
index 5d106b5c075..bdc0f04b56b 100644
--- a/lib/gitlab/i18n.rb
+++ b/lib/gitlab/i18n.rb
@@ -17,7 +17,8 @@ module Gitlab
       'it' => 'Italiano',
       'uk' => 'Українська',
       'ja' => '日本語',
-      'ko' => '한국어'
+      'ko' => '한국어',
+      'nl_NL' => 'Nederlands'
     }.freeze
 
     def available_locales
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 36708078136..6857038dba8 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -9,12 +9,28 @@ module Gitlab
 
       def uncached_data
         license_usage_data.merge(system_usage_data)
+                          .merge(features_usage_data)
+                          .merge(components_usage_data)
       end
 
       def to_json(force_refresh: false)
         data(force_refresh: force_refresh).to_json
       end
 
+      def license_usage_data
+        usage_data = {
+          uuid: current_application_settings.uuid,
+          hostname: Gitlab.config.gitlab.host,
+          version: Gitlab::VERSION,
+          active_user_count: User.active.count,
+          recorded_at: Time.now,
+          mattermost_enabled: Gitlab.config.mattermost.enabled,
+          edition: 'CE'
+        }
+
+        usage_data
+      end
+
       def system_usage_data
         {
           counts: {
@@ -54,18 +70,28 @@ module Gitlab
         }
       end
 
-      def license_usage_data
-        usage_data = {
-          uuid: current_application_settings.uuid,
-          hostname: Gitlab.config.gitlab.host,
-          version: Gitlab::VERSION,
-          active_user_count: User.active.count,
-          recorded_at: Time.now,
-          mattermost_enabled: Gitlab.config.mattermost.enabled,
-          edition: 'CE'
+      def features_usage_data
+        features_usage_data_ce
+      end
+
+      def features_usage_data_ce
+        {
+          signup: current_application_settings.signup_enabled?,
+          ldap: Gitlab.config.ldap.enabled,
+          gravatar: current_application_settings.gravatar_enabled?,
+          omniauth: Gitlab.config.omniauth.enabled,
+          reply_by_email: Gitlab::IncomingEmail.enabled?,
+          container_registry: Gitlab.config.registry.enabled,
+          gitlab_shared_runners: Gitlab.config.gitlab_ci.shared_runners_enabled
         }
+      end
 
-        usage_data
+      def components_usage_data
+        {
+          gitlab_pages: { enabled: Gitlab.config.pages.enabled, version: Gitlab::Pages::VERSION },
+          git: { version: Gitlab::Git.version },
+          database: { adapter: Gitlab::Database.adapter_name, version: Gitlab::Database.version }
+        }
       end
 
       def services_usage
diff --git a/lib/system_check/incoming_email/imap_authentication_check.rb b/lib/system_check/incoming_email/imap_authentication_check.rb
index dee108d987b..e55bea86d3f 100644
--- a/lib/system_check/incoming_email/imap_authentication_check.rb
+++ b/lib/system_check/incoming_email/imap_authentication_check.rb
@@ -4,22 +4,17 @@ module SystemCheck
       set_name 'IMAP server credentials are correct?'
 
       def check?
-        if mailbox_config
-          begin
-            imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl])
-            imap.starttls if config[:start_tls]
-            imap.login(config[:email], config[:password])
-            connected = true
-          rescue
-            connected = false
-          end
+        if config
+          try_connect_imap
+        else
+          @error = "#{mail_room_config_path} does not have mailboxes setup"
+          false
         end
-
-        connected
       end
 
       def show_error
         try_fixing_it(
+          "An error occurred: #{@error.class}: #{@error.message}",
           'Check that the information in config/gitlab.yml is correct'
         )
         for_more_information(
@@ -30,15 +25,31 @@ module SystemCheck
 
       private
 
-      def mailbox_config
-        return @config if @config
+      def try_connect_imap
+        imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl])
+        imap.starttls if config[:start_tls]
+        imap.login(config[:email], config[:password])
+        true
+      rescue => error
+        @error = error
+        false
+      end
+
+      def config
+        @config ||= load_config
+      end
+
+      def mail_room_config_path
+        @mail_room_config_path ||=
+          Rails.root.join('config', 'mail_room.yml').to_s
+      end
 
-        config_path = Rails.root.join('config', 'mail_room.yml').to_s
-        erb = ERB.new(File.read(config_path))
-        erb.filename = config_path
+      def load_config
+        erb = ERB.new(File.read(mail_room_config_path))
+        erb.filename = mail_room_config_path
         config_file = YAML.load(erb.result)
 
-        @config = config_file[:mailboxes]&.first
+        config_file.dig(:mailboxes, 0)
       end
     end
   end
diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake
index 7ccda04a35f..3eade7bf553 100644
--- a/lib/tasks/gitlab/dev.rake
+++ b/lib/tasks/gitlab/dev.rake
@@ -13,7 +13,10 @@ namespace :gitlab do
           args
         end
 
-      if Gitlab::EeCompatCheck.new(opts || {}).check
+      if File.basename(Rails.root) == 'gitlab-ee'
+        puts "Skipping EE projects"
+        exit 0
+      elsif Gitlab::EeCompatCheck.new(opts || {}).check
         exit 0
       else
         exit 1