Merge branch 'master' into issue-discussions-refactor

* master: (481 commits)
  Make sure that we have author and committer
  disable file upload button while uploading
  Fix bar chart does not display label at hour 0
  Fixed activity not loading on project homepage
  Expose noteable_iid in Note
  Fix fly-out width when it has long items
  Add a test to show that threshold 40 would corrupt
  Add changelog entry
  Raise encoding confidence threshold to 50
  Fix the /projects/:id/repository/commits endpoint to handle dots in the ref name when the project full path contains a `/`
  Fix the /projects/:id/repository/tags endpoint to handle dots in the tag name when the project full path contains a `/`
  Add Italian translations of Pipeline Schedules
  Restrict InlineJavaScript for haml_lint to dev and test environment
  Incorporate Gitaly's CommitService.FindCommit RPC
  Move `deltas` and `diff_from_parents` logic to Gitlab::Git::Commit
  fix repo_edit_button_spec.js
  fix test failures in repo_preview_spec.js
  fix repo_loading_file_spec tests
  Refactor Gitlab::Git::Commit to include a repository
  use 100vh instead of flip flopping between the two - works on all suported browsers
  ...

50 files changed, 1144 insertions, 470 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 982a2b88d62..94df543853b 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -95,6 +95,7 @@ module API
     mount ::API::Boards
     mount ::API::Branches
     mount ::API::BroadcastMessages
+    mount ::API::CircuitBreakers
     mount ::API::Commits
     mount ::API::CommitStatuses
     mount ::API::DeployKeys
diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb
new file mode 100644
index 00000000000..118883f5ea5
--- /dev/null
+++ b/lib/api/circuit_breakers.rb
@@ -0,0 +1,50 @@
+module API
+  class CircuitBreakers < Grape::API
+    before { authenticated_as_admin! }
+
+    resource :circuit_breakers do
+      params do
+        requires :type,
+                 type: String,
+                 desc: "The type of circuitbreaker",
+                 values: ['repository_storage']
+      end
+      resource ':type' do
+        namespace '', requirements: { type: 'repository_storage' } do
+          helpers do
+            def failing_storage_health
+              @failing_storage_health ||= Gitlab::Git::Storage::Health.for_failing_storages
+            end
+
+            def storage_health
+              @failing_storage_health ||= Gitlab::Git::Storage::Health.for_all_storages
+            end
+          end
+
+          desc 'Get all failing git storages' do
+            detail 'This feature was introduced in GitLab 9.5'
+            success Entities::RepositoryStorageHealth
+          end
+          get do
+            present storage_health, with: Entities::RepositoryStorageHealth
+          end
+
+          desc 'Get all failing git storages' do
+            detail 'This feature was introduced in GitLab 9.5'
+            success Entities::RepositoryStorageHealth
+          end
+          get 'failing' do
+            present failing_storage_health, with: Entities::RepositoryStorageHealth
+          end
+
+          desc 'Reset all storage failures and open circuitbreaker' do
+            detail 'This feature was introduced in GitLab 9.5'
+          end
+          delete do
+            Gitlab::Git::Storage::CircuitBreaker.reset_all!
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index bcb842b9211..ea78737288a 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -4,13 +4,14 @@ module API
   class Commits < Grape::API
     include PaginationParams
 
-    before { authenticate! }
+    COMMIT_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(sha: API::NO_SLASH_URL_PART_REGEX)
+
     before { authorize! :download_code, user_project }
 
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: { id: %r{[^/]+} } do
+    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
       desc 'Get a project repository commits' do
         success Entities::RepoCommit
       end
@@ -21,7 +22,7 @@ module API
         optional :path,     type: String, desc: 'The file path'
         use :pagination
       end
-      get ":id/repository/commits" do
+      get ':id/repository/commits' do
         path   = params[:path]
         before = params[:until]
         after  = params[:since]
@@ -53,16 +54,19 @@ module API
         detail 'This feature was introduced in GitLab 8.13'
       end
       params do
-        requires :branch, type: String, desc: 'The name of branch'
+        requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.'
         requires :commit_message, type: String, desc: 'Commit message'
         requires :actions, type: Array[Hash], desc: 'Actions to perform in commit'
+        optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from'
         optional :author_email, type: String, desc: 'Author email for commit'
         optional :author_name, type: String, desc: 'Author name for commit'
       end
-      post ":id/repository/commits" do
+      post ':id/repository/commits' do
         authorize! :push_code, user_project
 
-        attrs = declared_params.merge(start_branch: declared_params[:branch], branch_name: declared_params[:branch])
+        attrs = declared_params
+        attrs[:branch_name] = attrs.delete(:branch)
+        attrs[:start_branch] ||= attrs[:branch_name]
 
         result = ::Files::MultiService.new(user_project, current_user, attrs).execute
 
@@ -76,42 +80,42 @@ module API
 
       desc 'Get a specific commit of a project' do
         success Entities::RepoCommitDetail
-        failure [[404, 'Not Found']]
+        failure [[404, 'Commit Not Found']]
       end
       params do
         requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag'
       end
-      get ":id/repository/commits/:sha" do
+      get ':id/repository/commits/:sha', requirements: COMMIT_ENDPOINT_REQUIREMENTS do
         commit = user_project.commit(params[:sha])
 
-        not_found! "Commit" unless commit
+        not_found! 'Commit' unless commit
 
         present commit, with: Entities::RepoCommitDetail
       end
 
       desc 'Get the diff for a specific commit of a project' do
-        failure [[404, 'Not Found']]
+        failure [[404, 'Commit Not Found']]
       end
       params do
         requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag'
       end
-      get ":id/repository/commits/:sha/diff" do
+      get ':id/repository/commits/:sha/diff', requirements: COMMIT_ENDPOINT_REQUIREMENTS do
         commit = user_project.commit(params[:sha])
 
-        not_found! "Commit" unless commit
+        not_found! 'Commit' unless commit
 
         commit.raw_diffs.to_a
       end
 
       desc "Get a commit's comments" do
         success Entities::CommitNote
-        failure [[404, 'Not Found']]
+        failure [[404, 'Commit Not Found']]
       end
       params do
         use :pagination
         requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag'
       end
-      get ':id/repository/commits/:sha/comments' do
+      get ':id/repository/commits/:sha/comments', requirements: COMMIT_ENDPOINT_REQUIREMENTS do
         commit = user_project.commit(params[:sha])
 
         not_found! 'Commit' unless commit
@@ -125,10 +129,10 @@ module API
         success Entities::RepoCommit
       end
       params do
-        requires :sha, type: String, desc: 'A commit sha to be cherry picked'
+        requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag to be cherry picked'
         requires :branch, type: String, desc: 'The name of the branch'
       end
-      post ':id/repository/commits/:sha/cherry_pick' do
+      post ':id/repository/commits/:sha/cherry_pick', requirements: COMMIT_ENDPOINT_REQUIREMENTS do
         authorize! :push_code, user_project
 
         commit = user_project.commit(params[:sha])
@@ -157,7 +161,7 @@ module API
         success Entities::CommitNote
       end
       params do
-        requires :sha, type: String, regexp: /\A\h{6,40}\z/, desc: "The commit's SHA"
+        requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag on which to post a comment'
         requires :note, type: String, desc: 'The text of the comment'
         optional :path, type: String, desc: 'The file path'
         given :path do
@@ -165,7 +169,7 @@ module API
           requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line'
         end
       end
-      post ':id/repository/commits/:sha/comments' do
+      post ':id/repository/commits/:sha/comments', requirements: COMMIT_ENDPOINT_REQUIREMENTS do
         commit = user_project.commit(params[:sha])
         not_found! 'Commit' unless commit
 
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 29733481e2f..225879d94ac 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -66,13 +66,6 @@ module API
       expose :job_events
     end
 
-    class BasicProjectDetails < Grape::Entity
-      expose :id
-      expose :http_url_to_repo, :web_url
-      expose :name, :name_with_namespace
-      expose :path, :path_with_namespace
-    end
-
     class SharedGroup < Grape::Entity
       expose :group_id
       expose :group_name do |group_link, options|
@@ -81,7 +74,16 @@ module API
       expose :group_access, as: :group_access_level
     end
 
-    class Project < Grape::Entity
+    class BasicProjectDetails < Grape::Entity
+      expose :id, :description, :default_branch, :tag_list
+      expose :ssh_url_to_repo, :http_url_to_repo, :web_url
+      expose :name, :name_with_namespace
+      expose :path, :path_with_namespace
+      expose :star_count, :forks_count
+      expose :created_at, :last_activity_at
+    end
+
+    class Project < BasicProjectDetails 
       include ::API::Helpers::RelatedResourcesHelpers
 
       expose :_links do
@@ -114,12 +116,9 @@ module API
         end
       end
 
-      expose :id, :description, :default_branch, :tag_list
       expose :archived?, as: :archived
-      expose :visibility, :ssh_url_to_repo, :http_url_to_repo, :web_url
+      expose :visibility
       expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group }
-      expose :name, :name_with_namespace
-      expose :path, :path_with_namespace
       expose :container_registry_enabled
 
       # Expose old field names with the new permissions methods to keep API compatible
@@ -129,7 +128,6 @@ module API
       expose(:jobs_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) }
       expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) }
 
-      expose :created_at, :last_activity_at
       expose :shared_runners_enabled
       expose :lfs_enabled?, as: :lfs_enabled
       expose :creator_id
@@ -140,7 +138,6 @@ module API
       expose :avatar_url do |user, options|
         user.avatar_url(only_path: false)
       end
-      expose :star_count, :forks_count
       expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) }
       expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
       expose :public_builds, as: :public_jobs
@@ -457,6 +454,9 @@ module API
     end
 
     class Note < Grape::Entity
+      # Only Issue and MergeRequest have iid
+      NOTEABLE_TYPES_WITH_IID = %w(Issue MergeRequest).freeze
+
       expose :id
       expose :note, as: :body
       expose :attachment_identifier, as: :attachment
@@ -464,6 +464,9 @@ module API
       expose :created_at, :updated_at
       expose :system?, as: :system
       expose :noteable_id, :noteable_type
+
+      # Avoid N+1 queries as much as possible
+      expose(:noteable_iid) { |note| note.noteable.iid if NOTEABLE_TYPES_WITH_IID.include?(note.noteable_type) }
     end
 
     class AwardEmoji < Grape::Entity
@@ -702,7 +705,7 @@ module API
     class RepoTag < Grape::Entity
       expose :name, :message
 
-      expose :commit do |repo_tag, options|
+      expose :commit, using: Entities::RepoCommit do |repo_tag, options|
         options[:project].repository.commit(repo_tag.dereferenced_target)
       end
 
@@ -954,5 +957,11 @@ module API
       expose :ip_address
       expose :submitted, as: :akismet_submitted
     end
+
+    class RepositoryStorageHealth < Grape::Entity
+      expose :storage_name
+      expose :failing_on_hosts
+      expose :total_failures
+    end
   end
 end
diff --git a/lib/api/files.rb b/lib/api/files.rb
index 521287ee2b4..450334fee84 100644
--- a/lib/api/files.rb
+++ b/lib/api/files.rb
@@ -4,7 +4,7 @@ module API
       def commit_params(attrs)
         {
           file_path: attrs[:file_path],
-          start_branch: attrs[:branch],
+          start_branch: attrs[:start_branch] || attrs[:branch],
           branch_name: attrs[:branch],
           commit_message: attrs[:commit_message],
           file_content: attrs[:content],
@@ -37,8 +37,9 @@ module API
 
       params :simple_file_params do
         requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb'
-        requires :branch, type: String, desc: 'The name of branch'
-        requires :commit_message, type: String, desc: 'Commit Message'
+        requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.'
+        requires :commit_message, type: String, desc: 'Commit message'
+        optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from'
         optional :author_email, type: String, desc: 'The email of the author'
         optional :author_name, type: String, desc: 'The name of the author'
       end
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index 633a858f8c7..1333747cced 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -2,19 +2,21 @@ module API
   class Tags < Grape::API
     include PaginationParams
 
+    TAG_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(tag_name: API::NO_SLASH_URL_PART_REGEX)
+
     before { authorize! :download_code, user_project }
 
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: { id: %r{[^/]+} } do
+    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
       desc 'Get a project repository tags' do
         success Entities::RepoTag
       end
       params do
         use :pagination
       end
-      get ":id/repository/tags" do
+      get ':id/repository/tags' do
         tags = ::Kaminari.paginate_array(user_project.repository.tags.sort_by(&:name).reverse)
         present paginate(tags), with: Entities::RepoTag, project: user_project
       end
@@ -25,7 +27,7 @@ module API
       params do
         requires :tag_name, type: String, desc: 'The name of the tag'
       end
-      get ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do
+      get ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS do
         tag = user_project.repository.find_tag(params[:tag_name])
         not_found!('Tag') unless tag
 
@@ -60,7 +62,7 @@ module API
       params do
         requires :tag_name, type: String, desc: 'The name of the tag'
       end
-      delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do
+      delete ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS do
         authorize_push_project
 
         result = ::Tags::DestroyService.new(user_project, current_user)
@@ -78,7 +80,7 @@ module API
         requires :tag_name,    type: String, desc: 'The name of the tag'
         requires :description, type: String, desc: 'Release notes with markdown support'
       end
-      post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do
+      post ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do
         authorize_push_project
 
         result = CreateReleaseService.new(user_project, current_user)
@@ -98,7 +100,7 @@ module API
         requires :tag_name,    type: String, desc: 'The name of the tag'
         requires :description, type: String, desc: 'Release notes with markdown support'
       end
-      put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do
+      put ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do
         authorize_push_project
 
         result = UpdateReleaseService.new(user_project, current_user)
diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index 685b43605ae..ef4578aabd6 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -54,42 +54,42 @@ module Banzai
         self.class.references_in(*args, &block)
       end
 
+      # Implement in child class
+      # Example: project.merge_requests.find
       def find_object(project, id)
-        # Implement in child class
-        # Example: project.merge_requests.find
       end
 
-      def find_object_cached(project, id)
-        if RequestStore.active?
-          cache = find_objects_cache[object_class][project.id]
+      # Override if the link reference pattern produces a different ID (global
+      # ID vs internal ID, for instance) to the regular reference pattern.
+      def find_object_from_link(project, id)
+        find_object(project, id)
+      end
 
-          get_or_set_cache(cache, id) { find_object(project, id) }
-        else
+      # Implement in child class
+      # Example: project_merge_request_url
+      def url_for_object(object, project)
+      end
+
+      def find_object_cached(project, id)
+        cached_call(:banzai_find_object, id, path: [object_class, project.id]) do
           find_object(project, id)
         end
       end
 
-      def project_from_ref_cached(ref)
-        if RequestStore.active?
-          cache = project_refs_cache
-
-          get_or_set_cache(cache, ref) { project_from_ref(ref) }
-        else
-          project_from_ref(ref)
+      def find_object_from_link_cached(project, id)
+        cached_call(:banzai_find_object_from_link, id, path: [object_class, project.id]) do
+          find_object_from_link(project, id)
         end
       end
 
-      def url_for_object(object, project)
-        # Implement in child class
-        # Example: project_merge_request_url
+      def project_from_ref_cached(ref)
+        cached_call(:banzai_project_refs, ref) do
+          project_from_ref(ref)
+        end
       end
 
       def url_for_object_cached(object, project)
-        if RequestStore.active?
-          cache = url_for_object_cache[object_class][project.id]
-
-          get_or_set_cache(cache, object) { url_for_object(object, project) }
-        else
+        cached_call(:banzai_url_for_object, object, path: [object_class, project.id]) do
           url_for_object(object, project)
         end
       end
@@ -120,7 +120,7 @@ module Banzai
 
               if link == inner_html && inner_html =~ /\A#{link_pattern}/
                 replace_link_node_with_text(node, link) do
-                  object_link_filter(inner_html, link_pattern)
+                  object_link_filter(inner_html, link_pattern, link_reference: true)
                 end
 
                 next
@@ -128,7 +128,7 @@ module Banzai
 
               if link =~ /\A#{link_pattern}\z/
                 replace_link_node_with_href(node, link) do
-                  object_link_filter(link, link_pattern, link_content: inner_html)
+                  object_link_filter(link, link_pattern, link_content: inner_html, link_reference: true)
                 end
 
                 next
@@ -146,15 +146,26 @@ module Banzai
       # text - String text to replace references in.
       # pattern - Reference pattern to match against.
       # link_content - Original content of the link being replaced.
+      # link_reference - True if this was using the link reference pattern,
+      #                  false otherwise.
       #
       # Returns a String with references replaced with links. All links
       # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling.
-      def object_link_filter(text, pattern, link_content: nil)
+      def object_link_filter(text, pattern, link_content: nil, link_reference: false)
         references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches|
           project_path = full_project_path(namespace_ref, project_ref)
           project = project_from_ref_cached(project_path)
 
-          if project && object = find_object_cached(project, id)
+          if project
+            object =
+              if link_reference
+                find_object_from_link_cached(project, id)
+              else
+                find_object_cached(project, id)
+              end
+          end
+
+          if object
             title = object_link_title(object)
             klass = reference_class(object_sym)
 
@@ -297,15 +308,17 @@ module Banzai
         RequestStore[:banzai_project_refs] ||= {}
       end
 
-      def find_objects_cache
-        RequestStore[:banzai_find_objects_cache] ||= Hash.new do |hash, key|
-          hash[key] = Hash.new { |h, k| h[k] = {} }
-        end
-      end
+      def cached_call(request_store_key, cache_key, path: [])
+        if RequestStore.active?
+          cache = RequestStore[request_store_key] ||= Hash.new do |hash, key|
+            hash[key] = Hash.new { |h, k| h[k] = {} }
+          end
 
-      def url_for_object_cache
-        RequestStore[:banzai_url_for_object] ||= Hash.new do |hash, key|
-          hash[key] = Hash.new { |h, k| h[k] = {} }
+          cache = cache.dig(*path) if path.any?
+
+          get_or_set_cache(cache, cache_key) { yield }
+        else
+          yield
         end
       end
 
diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb
index 45c033d32a8..4fc5f211e84 100644
--- a/lib/banzai/filter/milestone_reference_filter.rb
+++ b/lib/banzai/filter/milestone_reference_filter.rb
@@ -8,8 +8,15 @@ module Banzai
         Milestone
       end
 
+      # Links to project milestones contain the IID, but when we're handling
+      # 'regular' references, we need to use the global ID to disambiguate
+      # between group and project milestones.
       def find_object(project, id)
-        project.milestones.find_by(iid: id)
+        find_milestone_with_finder(project, id: id)
+      end
+
+      def find_object_from_link(project, iid)
+        find_milestone_with_finder(project, iid: iid)
       end
 
       def references_in(text, pattern = Milestone.reference_pattern)
@@ -22,7 +29,7 @@ module Banzai
           milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name])
 
           if milestone
-            yield match, milestone.iid, $~[:project], $~[:namespace], $~
+            yield match, milestone.id, $~[:project], $~[:namespace], $~
           else
             match
           end
@@ -36,7 +43,8 @@ module Banzai
         return unless project
 
         milestone_params = milestone_params(milestone_id, milestone_name)
-        project.milestones.find_by(milestone_params)
+
+        find_milestone_with_finder(project, milestone_params)
       end
 
       def milestone_params(iid, name)
@@ -47,15 +55,27 @@ module Banzai
         end
       end
 
+      def find_milestone_with_finder(project, params)
+        finder_params = { project_ids: [project.id], order: nil }
+
+        # We don't support IID lookups for group milestones, because IIDs can
+        # clash between group and project milestones.
+        if project.group && !params[:iid]
+          finder_params[:group_ids] = [project.group.id]
+        end
+
+        MilestonesFinder.new(finder_params).execute.find_by(params)
+      end
+
       def url_for_object(milestone, project)
-        h = Gitlab::Routing.url_helpers
-        h.project_milestone_url(project, milestone,
-                                        only_path: context[:only_path])
+        Gitlab::Routing
+          .url_helpers
+          .milestone_url(milestone, only_path: context[:only_path])
       end
 
       def object_link_text(object, matches)
         milestone_link = escape_once(super)
-        reference = object.project.to_reference(project)
+        reference = object.project&.to_reference(project)
 
         if reference.present?
           "#{milestone_link} <i>in #{reference}</i>".html_safe
diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb
index b5c615da4e3..56afd1f1392 100644
--- a/lib/declarative_policy/runner.rb
+++ b/lib/declarative_policy/runner.rb
@@ -76,6 +76,8 @@ module DeclarativePolicy
       @state = State.new
 
       steps_by_score do |step, score|
+        return if !debug && @state.prevented?
+
         passed = nil
         case step.action
         when :enable then
@@ -93,10 +95,7 @@ module DeclarativePolicy
           # been prevented.
           unless @state.prevented?
             passed = step.pass?
-            if passed
-              @state.prevent!
-              return unless debug
-            end
+            @state.prevent! if passed
           end
 
           debug << inspect_step(step, score, passed) if debug
@@ -141,13 +140,14 @@ module DeclarativePolicy
       end
 
       steps = Set.new(@steps)
+      remaining_enablers = steps.count { |s| s.enable? }
 
       loop do
         return if steps.empty?
 
         # if the permission hasn't yet been enabled and we only have
         # prevent steps left, we short-circuit the state here
-        @state.prevent! if !@state.enabled? && steps.all?(&:prevent?)
+        @state.prevent! if !@state.enabled? && remaining_enablers == 0
 
         lowest_score = Float::INFINITY
         next_step = nil
@@ -162,6 +162,8 @@ module DeclarativePolicy
 
         steps.delete(next_step)
 
+        remaining_enablers -= 1 if next_step.enable?
+
         yield next_step, lowest_score
       end
     end
diff --git a/lib/github/client.rb b/lib/github/client.rb
index e65d908d232..9c476df7d46 100644
--- a/lib/github/client.rb
+++ b/lib/github/client.rb
@@ -1,13 +1,16 @@
 module Github
   class Client
+    TIMEOUT = 60
+
     attr_reader :connection, :rate_limit
 
     def initialize(options)
-      @connection = Faraday.new(url: options.fetch(:url)) do |faraday|
-        faraday.options.open_timeout = options.fetch(:timeout, 60)
-        faraday.options.timeout = options.fetch(:timeout, 60)
+      @connection = Faraday.new(url: options.fetch(:url, root_endpoint)) do |faraday|
+        faraday.options.open_timeout = options.fetch(:timeout, TIMEOUT)
+        faraday.options.timeout = options.fetch(:timeout, TIMEOUT)
         faraday.authorization 'token', options.fetch(:token)
         faraday.adapter :net_http
+        faraday.ssl.verify = verify_ssl
       end
 
       @rate_limit = RateLimit.new(connection)
@@ -19,5 +22,32 @@ module Github
 
       Github::Response.new(connection.get(url, query))
     end
+
+    private
+
+    def root_endpoint
+      custom_endpoint || github_endpoint
+    end
+
+    def custom_endpoint
+      github_omniauth_provider.dig('args', 'client_options', 'site')
+    end
+
+    def verify_ssl
+      # If there is no config, we're connecting to github.com
+      # and we should verify ssl.
+      github_omniauth_provider.fetch('verify_ssl', true)
+    end
+
+    def github_endpoint
+      OmniAuth::Strategies::GitHub.default_options[:client_options][:site]
+    end
+
+    def github_omniauth_provider
+      @github_omniauth_provider ||=
+        Gitlab.config.omniauth.providers
+              .find { |provider| provider.name == 'github' }
+              .to_h
+    end
   end
 end
diff --git a/lib/github/import.rb b/lib/github/import.rb
index cea4be5460b..4cc01593ef4 100644
--- a/lib/github/import.rb
+++ b/lib/github/import.rb
@@ -41,13 +41,16 @@ module Github
       self.reset_callbacks :validate
     end
 
-    attr_reader :project, :repository, :repo, :options, :errors, :cached, :verbose
+    attr_reader :project, :repository, :repo, :repo_url, :wiki_url,
+                :options, :errors, :cached, :verbose
 
-    def initialize(project, options)
+    def initialize(project, options = {})
       @project = project
       @repository = project.repository
       @repo = project.import_source
-      @options = options
+      @repo_url = project.import_url
+      @wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git')
+      @options = options.reverse_merge(token: project.import_data&.credentials&.fetch(:user))
       @verbose = options.fetch(:verbose, false)
       @cached  = Hash.new { |hash, key| hash[key] = Hash.new }
       @errors  = []
@@ -65,6 +68,8 @@ module Github
       fetch_pull_requests
       puts 'Fetching issues...'.color(:aqua) if verbose
       fetch_issues
+      puts 'Fetching releases...'.color(:aqua) if verbose
+      fetch_releases
       puts 'Cloning wiki repository...'.color(:aqua) if verbose
       fetch_wiki_repository
       puts 'Expiring repository cache...'.color(:aqua) if verbose
@@ -72,6 +77,7 @@ module Github
 
       true
     rescue Github::RepositoryFetchError
+      expire_repository_cache
       false
     ensure
       keep_track_of_errors
@@ -81,23 +87,21 @@ module Github
 
     def fetch_repository
       begin
-        project.create_repository unless project.repository.exists?
-        project.repository.add_remote('github', "https://#{options.fetch(:token)}@github.com/#{repo}.git")
+        project.ensure_repository
+        project.repository.add_remote('github', repo_url)
         project.repository.set_remote_as_mirror('github')
         project.repository.fetch_remote('github', forced: true)
-      rescue Gitlab::Shell::Error => e
-        error(:project, "https://github.com/#{repo}.git", e.message)
+      rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e
+        error(:project, repo_url, e.message)
         raise Github::RepositoryFetchError
       end
     end
 
     def fetch_wiki_repository
-      wiki_url  = "https://#{options.fetch(:token)}@github.com/#{repo}.wiki.git"
-      wiki_path = "#{project.full_path}.wiki"
+      return if project.wiki.repository_exists?
 
-      unless project.wiki.repository_exists?
-        gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url)
-      end
+      wiki_path = "#{project.disk_path}.wiki"
+      gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url)
     rescue Gitlab::Shell::Error => e
       # GitHub error message when the wiki repo has not been created,
       # this means that repo has wiki enabled, but have no pages. So,
@@ -309,7 +313,7 @@ module Github
           next unless representation.valid?
 
           release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag)
-          next unless relese.new_record?
+          next unless release.new_record?
 
           begin
             release.description = representation.description
@@ -337,7 +341,7 @@ module Github
 
     def user_id(user, fallback_id = nil)
       return unless user.present?
-      return cached[:user_ids][user.id] if cached[:user_ids].key?(user.id)
+      return cached[:user_ids][user.id] if cached[:user_ids][user.id].present?
 
       gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email)
 
@@ -367,7 +371,7 @@ module Github
     end
 
     def expire_repository_cache
-      repository.expire_content_cache
+      repository.expire_content_cache if project.repository_exists?
     end
 
     def keep_track_of_errors
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 9bed81e7327..7d3aa532750 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -218,7 +218,8 @@ module Gitlab
       def full_authentication_abilities
         read_authentication_abilities + [
           :push_code,
-          :create_container_image
+          :create_container_image,
+          :admin_container_image
         ]
       end
       alias_method :api_scope_authentication_abilities, :full_authentication_abilities
diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb
index 1611eba31da..d671867e7c7 100644
--- a/lib/gitlab/conflict/file_collection.rb
+++ b/lib/gitlab/conflict/file_collection.rb
@@ -77,8 +77,8 @@ EOM
 
       def initialize(merge_request, project)
         @merge_request = merge_request
-        @our_commit = merge_request.source_branch_head.raw.raw_commit
-        @their_commit = merge_request.target_branch_head.raw.raw_commit
+        @our_commit = merge_request.source_branch_head.raw.rugged_commit
+        @their_commit = merge_request.target_branch_head.raw.rugged_commit
         @project = project
       end
     end
diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
index b260822788d..2479b4a7706 100644
--- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
@@ -54,7 +54,7 @@ module Gitlab
       end
 
       def serialize_commit(event, commit, query)
-        commit = Commit.new(Gitlab::Git::Commit.new(commit.to_hash), @project)
+        commit = Commit.from_hash(commit.to_hash, @project)
 
         AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit)
       end
diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb
new file mode 100644
index 00000000000..dfd17e35707
--- /dev/null
+++ b/lib/gitlab/daemon.rb
@@ -0,0 +1,62 @@
+module Gitlab
+  class Daemon
+    def self.initialize_instance(*args)
+      raise "#{name} singleton instance already initialized" if @instance
+      @instance = new(*args)
+      Kernel.at_exit(&@instance.method(:stop))
+      @instance
+    end
+
+    def self.instance
+      @instance ||= initialize_instance
+    end
+
+    attr_reader :thread
+
+    def thread?
+      !thread.nil?
+    end
+
+    def initialize
+      @mutex = Mutex.new
+    end
+
+    def enabled?
+      true
+    end
+
+    def start
+      return unless enabled?
+
+      @mutex.synchronize do
+        return thread if thread?
+
+        @thread = Thread.new { start_working }
+      end
+    end
+
+    def stop
+      @mutex.synchronize do
+        return unless thread?
+
+        stop_working
+
+        if thread
+          thread.wakeup if thread.alive?
+          thread.join
+          @thread = nil
+        end
+      end
+    end
+
+    private
+
+    def start_working
+      raise NotImplementedError
+    end
+
+    def stop_working
+      # no-ops
+    end
+  end
+end
diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb
index 781f9c56a42..8ddc91e341d 100644
--- a/lib/gitlab/encoding_helper.rb
+++ b/lib/gitlab/encoding_helper.rb
@@ -11,7 +11,7 @@ module Gitlab
     # obscure encoding with low confidence.
     # There is a lot more info with this merge request:
     # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
-    ENCODING_CONFIDENCE_THRESHOLD = 40
+    ENCODING_CONFIDENCE_THRESHOLD = 50
 
     def encode!(message)
       return nil unless message.respond_to? :force_encoding
diff --git a/lib/gitlab/environment.rb b/lib/gitlab/environment.rb
new file mode 100644
index 00000000000..5e0dd6e7859
--- /dev/null
+++ b/lib/gitlab/environment.rb
@@ -0,0 +1,7 @@
+module Gitlab
+  module Environment
+    def self.hostname
+      @hostname ||= ENV['HOSTNAME'] || Socket.gethostname
+    end
+  end
+end
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
index 8dbe25e55f6..31effdba292 100644
--- a/lib/gitlab/git/blame.rb
+++ b/lib/gitlab/git/blame.rb
@@ -16,7 +16,7 @@ module Gitlab
       def each
         @blames.each do |blame|
           yield(
-            Gitlab::Git::Commit.new(blame.commit),
+            Gitlab::Git::Commit.new(@repo, blame.commit),
             blame.line
           )
         end
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index db6cfc9671f..77b81d2d437 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -20,66 +20,7 @@ module Gitlab
             if is_enabled
               find_by_gitaly(repository, sha, path)
             else
-              find_by_rugged(repository, sha, path)
-            end
-          end
-        end
-
-        def find_by_gitaly(repository, sha, path)
-          path = path.sub(/\A\/*/, '')
-          path = '/' if path.empty?
-          name = File.basename(path)
-          entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE)
-          return unless entry
-
-          case entry.type
-          when :COMMIT
-            new(
-              id: entry.oid,
-              name: name,
-              size: 0,
-              data: '',
-              path: path,
-              commit_id: sha
-            )
-          when :BLOB
-            new(
-              id: entry.oid,
-              name: name,
-              size: entry.size,
-              data: entry.data.dup,
-              mode: entry.mode.to_s(8),
-              path: path,
-              commit_id: sha,
-              binary: binary?(entry.data)
-            )
-          end
-        end
-
-        def find_by_rugged(repository, sha, path)
-          commit = repository.lookup(sha)
-          root_tree = commit.tree
-
-          blob_entry = find_entry_by_path(repository, root_tree.oid, path)
-
-          return nil unless blob_entry
-
-          if blob_entry[:type] == :commit
-            submodule_blob(blob_entry, path, sha)
-          else
-            blob = repository.lookup(blob_entry[:oid])
-
-            if blob
-              new(
-                id: blob.oid,
-                name: blob_entry[:name],
-                size: blob.size,
-                data: blob.content(MAX_DATA_DISPLAY_SIZE),
-                mode: blob_entry[:filemode].to_s(8),
-                path: path,
-                commit_id: sha,
-                binary: blob.binary?
-              )
+              find_by_rugged(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE)
             end
           end
         end
@@ -109,6 +50,21 @@ module Gitlab
           detect && detect[:type] == :binary
         end
 
+        # Returns an array of Blob instances, specified in blob_references as
+        # [[commit_sha, path], [commit_sha, path], ...]. If blob_size_limit < 0 then the
+        # full blob contents are returned. If blob_size_limit >= 0 then each blob will
+        # contain no more than limit bytes in its data attribute.
+        # 
+        # Keep in mind that this method may allocate a lot of memory. It is up
+        # to the caller to limit the number of blobs and blob_size_limit.
+        #
+        def batch(repository, blob_references, blob_size_limit: nil)
+          blob_size_limit ||= MAX_DATA_DISPLAY_SIZE
+          blob_references.map do |sha, path|
+            find_by_rugged(repository, sha, path, limit: blob_size_limit)
+          end
+        end
+
         private
 
         # Recursive search of blob id by path
@@ -153,6 +109,66 @@ module Gitlab
             commit_id: sha
           )
         end
+
+        def find_by_gitaly(repository, sha, path)
+          path = path.sub(/\A\/*/, '')
+          path = '/' if path.empty?
+          name = File.basename(path)
+          entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE)
+          return unless entry
+
+          case entry.type
+          when :COMMIT
+            new(
+              id: entry.oid,
+              name: name,
+              size: 0,
+              data: '',
+              path: path,
+              commit_id: sha
+            )
+          when :BLOB
+            new(
+              id: entry.oid,
+              name: name,
+              size: entry.size,
+              data: entry.data.dup,
+              mode: entry.mode.to_s(8),
+              path: path,
+              commit_id: sha,
+              binary: binary?(entry.data)
+            )
+          end
+        end
+
+        def find_by_rugged(repository, sha, path, limit:)
+          commit = repository.lookup(sha)
+          root_tree = commit.tree
+
+          blob_entry = find_entry_by_path(repository, root_tree.oid, path)
+
+          return nil unless blob_entry
+
+          if blob_entry[:type] == :commit
+            submodule_blob(blob_entry, path, sha)
+          else
+            blob = repository.lookup(blob_entry[:oid])
+
+            if blob
+              new(
+                id: blob.oid,
+                name: blob_entry[:name],
+                size: blob.size,
+                # Rugged::Blob#content is expensive; don't call it if we don't have to.
+                data: limit.zero? ? '' : blob.content(limit),
+                mode: blob_entry[:filemode].to_s(8),
+                path: path,
+                commit_id: sha,
+                binary: blob.binary?
+              )
+            end
+          end
+        end
       end
 
       def initialize(options)
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index ca7e3a7c4be..9256663f454 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -14,7 +14,7 @@ module Gitlab
 
       attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator
 
-      delegate :tree, to: :raw_commit
+      delegate :tree, to: :rugged_commit
 
       def ==(other)
         return false unless other.is_a?(Gitlab::Git::Commit)
@@ -50,19 +50,29 @@ module Gitlab
         #
         # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/321
         def find(repo, commit_id = "HEAD")
+          # Already a commit?
           return commit_id if commit_id.is_a?(Gitlab::Git::Commit)
-          return decorate(commit_id) if commit_id.is_a?(Rugged::Commit)
 
-          obj = if commit_id.is_a?(String)
-                  repo.rev_parse_target(commit_id)
-                else
-                  Gitlab::Git::Ref.dereference_object(commit_id)
-                end
+          # A rugged reference?
+          commit_id = Gitlab::Git::Ref.dereference_object(commit_id)
+          return decorate(repo, commit_id) if commit_id.is_a?(Rugged::Commit)
 
-          return nil unless obj.is_a?(Rugged::Commit)
+          # Some weird thing?
+          return nil unless commit_id.is_a?(String)
 
-          decorate(obj)
-        rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, Gitlab::Git::Repository::NoRepository
+          commit = repo.gitaly_migrate(:find_commit) do |is_enabled|
+            if is_enabled
+              repo.gitaly_commit_client.find_commit(commit_id)
+            else
+              obj = repo.rev_parse_target(commit_id)
+
+              obj.is_a?(Rugged::Commit) ? obj : nil
+            end
+          end
+
+          decorate(repo, commit) if commit
+        rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError,
+               Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository
           nil
         end
 
@@ -102,7 +112,7 @@ module Gitlab
             if is_enabled
               repo.gitaly_commit_client.between(base, head)
             else
-              repo.commits_between(base, head).map { |c| decorate(c) }
+              repo.rugged_commits_between(base, head).map { |c| decorate(repo, c) }
             end
           end
         rescue Rugged::ReferenceError
@@ -169,7 +179,7 @@ module Gitlab
           offset = actual_options[:skip]
           limit = actual_options[:max_count]
           walker.each(offset: offset, limit: limit) do |commit|
-            commits.push(decorate(commit))
+            commits.push(decorate(repo, commit))
           end
 
           walker.reset
@@ -183,27 +193,8 @@ module Gitlab
           Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options)
         end
 
-        def decorate(commit, ref = nil)
-          Gitlab::Git::Commit.new(commit, ref)
-        end
-
-        # Returns a diff object for the changes introduced by +rugged_commit+.
-        # If +rugged_commit+ doesn't have a parent, then the diff is between
-        # this commit and an empty repo.  See Repository#diff for the keys
-        # allowed in the +options+ hash.
-        def diff_from_parent(rugged_commit, options = {})
-          options ||= {}
-          break_rewrites = options[:break_rewrites]
-          actual_options = Gitlab::Git::Diff.filter_diff_options(options)
-
-          diff = if rugged_commit.parents.empty?
-                   rugged_commit.diff(actual_options.merge(reverse: true))
-                 else
-                   rugged_commit.parents[0].diff(rugged_commit, actual_options)
-                 end
-
-          diff.find_similar!(break_rewrites: break_rewrites)
-          diff
+        def decorate(repository, commit, ref = nil)
+          Gitlab::Git::Commit.new(repository, commit, ref)
         end
 
         # Returns the `Rugged` sorting type constant for one or more given
@@ -221,7 +212,7 @@ module Gitlab
         end
       end
 
-      def initialize(raw_commit, head = nil)
+      def initialize(repository, raw_commit, head = nil)
         raise "Nil as raw commit passed" unless raw_commit
 
         case raw_commit
@@ -229,12 +220,13 @@ module Gitlab
           init_from_hash(raw_commit)
         when Rugged::Commit
           init_from_rugged(raw_commit)
-        when Gitlab::GitalyClient::Commit
+        when Gitaly::GitCommit
           init_from_gitaly(raw_commit)
         else
           raise "Invalid raw commit type: #{raw_commit.class}"
         end
 
+        @repository = repository
         @head = head
       end
 
@@ -269,19 +261,50 @@ module Gitlab
       #
       # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324
       def to_diff
-        diff_from_parent.patch
+        rugged_diff_from_parent.patch
       end
 
       # Returns a diff object for the changes from this commit's first parent.
       # If there is no parent, then the diff is between this commit and an
-      # empty repo.  See Repository#diff for keys allowed in the +options+
+      # empty repo. See Repository#diff for keys allowed in the +options+
       # hash.
       def diff_from_parent(options = {})
-        Commit.diff_from_parent(raw_commit, options)
+        Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled|
+          if is_enabled
+            @repository.gitaly_commit_client.diff_from_parent(self, options)
+          else
+            rugged_diff_from_parent(options)
+          end
+        end
+      end
+
+      def rugged_diff_from_parent(options = {})
+        options ||= {}
+        break_rewrites = options[:break_rewrites]
+        actual_options = Gitlab::Git::Diff.filter_diff_options(options)
+
+        diff = if rugged_commit.parents.empty?
+                 rugged_commit.diff(actual_options.merge(reverse: true))
+               else
+                 rugged_commit.parents[0].diff(rugged_commit, actual_options)
+               end
+
+        diff.find_similar!(break_rewrites: break_rewrites)
+        diff
       end
 
       def deltas
-        @deltas ||= diff_from_parent.each_delta.map { |d| Gitlab::Git::Diff.new(d) }
+        @deltas ||= begin
+          deltas = Gitlab::GitalyClient.migrate(:commit_deltas) do |is_enabled|
+            if is_enabled
+              @repository.gitaly_commit_client.commit_deltas(self)
+            else
+              rugged_diff_from_parent.each_delta
+            end
+          end
+
+          deltas.map { |delta| Gitlab::Git::Diff.new(delta) }
+        end
       end
 
       def has_zero_stats?
@@ -309,14 +332,7 @@ module Gitlab
       end
 
       def parents
-        case raw_commit
-        when Rugged::Commit
-          raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) }
-        when Gitlab::GitalyClient::Commit
-          parent_ids.map { |oid| self.class.find(raw_commit.repository, oid) }.compact
-        else
-          raise NotImplementedError, "commit source doesn't support #parents"
-        end
+        parent_ids.map { |oid| self.class.find(@repository, oid) }.compact
       end
 
       # Get the gpg signature of this commit.
@@ -334,7 +350,7 @@ module Gitlab
 
       def to_patch(options = {})
         begin
-          raw_commit.to_mbox(options)
+          rugged_commit.to_mbox(options)
         rescue Rugged::InvalidError => ex
           if ex.message =~ /commit \w+ is a merge commit/i
             'Patch format is not currently supported for merge commits.'
@@ -382,6 +398,14 @@ module Gitlab
         encode! @committer_email
       end
 
+      def rugged_commit
+        @rugged_commit ||= if raw_commit.is_a?(Rugged::Commit)
+                             raw_commit
+                           else
+                             @repository.rev_parse_target(id)
+                           end
+      end
+
       private
 
       def init_from_hash(hash)
@@ -415,10 +439,10 @@ module Gitlab
         # subject from the message to make it clearer when there's one
         # available but not the other.
         @message = (commit.body.presence || commit.subject).dup
-        @authored_date = Time.at(commit.author.date.seconds)
+        @authored_date = Time.at(commit.author.date.seconds).utc
         @author_name = commit.author.name.dup
         @author_email = commit.author.email.dup
-        @committed_date = Time.at(commit.committer.date.seconds)
+        @committed_date = Time.at(commit.committer.date.seconds).utc
         @committer_name = commit.committer.name.dup
         @committer_email = commit.committer.email.dup
         @parent_ids = commit.parent_ids
diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb
index 57c29ad112c..00acb4763e9 100644
--- a/lib/gitlab/git/commit_stats.rb
+++ b/lib/gitlab/git/commit_stats.rb
@@ -16,7 +16,7 @@ module Gitlab
         @deletions = 0
         @total = 0
 
-        diff = commit.diff_from_parent
+        diff = commit.rugged_diff_from_parent
 
         diff.each_patch do |p|
           # TODO: Use the new Rugged convenience methods when they're released
diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb
index 87ed9c3ea26..6a601561c2a 100644
--- a/lib/gitlab/git/diff_collection.rb
+++ b/lib/gitlab/git/diff_collection.rb
@@ -28,7 +28,6 @@ module Gitlab
         @limits = self.class.collection_limits(options)
         @enforce_limits = !!options.fetch(:limits, true)
         @expanded = !!options.fetch(:expanded, true)
-        @from_gitaly = options.fetch(:from_gitaly, false)
 
         @line_count = 0
         @byte_count = 0
@@ -44,7 +43,7 @@ module Gitlab
         return if @iterator.nil?
 
         Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled|
-          if is_enabled && @from_gitaly
+          if is_enabled && @iterator.is_a?(Gitlab::GitalyClient::DiffStitcher)
             each_gitaly_patch(&block)
           else
             each_rugged_patch(&block)
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 734aed8fbc1..371f8797ff2 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -58,17 +58,18 @@ module Gitlab
         end
       end
 
-      # Alias to old method for compatibility
-      def raw
-        rugged
-      end
-
       def rugged
-        @rugged ||= Rugged::Repository.new(path, alternates: alternate_object_directories)
+        @rugged ||= circuit_breaker.perform do
+          Rugged::Repository.new(path, alternates: alternate_object_directories)
+        end
       rescue Rugged::RepositoryError, Rugged::OSError
         raise NoRepository.new('no repository for such path')
       end
 
+      def circuit_breaker
+        @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage)
+      end
+
       # Returns an Array of branch names
       # sorted by name ASC
       def branch_names
@@ -281,7 +282,14 @@ module Gitlab
 
       # Return repo size in megabytes
       def size
-        size = popen(%w(du -sk), path).first.strip.to_i
+        size = gitaly_migrate(:repository_size) do |is_enabled|
+          if is_enabled
+            size_by_gitaly
+          else
+            size_by_shelling_out
+          end
+        end
+
         (size.to_f / 1024).round(2)
       end
 
@@ -296,8 +304,24 @@ module Gitlab
       #     after: Time.new(2016, 4, 21, 14, 32, 10)
       #   )
       #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/446
       def log(options)
-        raw_log(options).map { |c| Commit.decorate(c) }
+        default_options = {
+          limit: 10,
+          offset: 0,
+          path: nil,
+          follow: false,
+          skip_merges: false,
+          disable_walk: false,
+          after: nil,
+          before: nil
+        }
+
+        options = default_options.merge(options)
+        options[:limit] ||= 0
+        options[:offset] ||= 0
+
+        raw_log(options).map { |c| Commit.decorate(self, c) }
       end
 
       def count_commits(options)
@@ -324,7 +348,9 @@ module Gitlab
       # Return a collection of Rugged::Commits between the two revspec arguments.
       # See http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for
       # a detailed list of valid arguments.
-      def commits_between(from, to)
+      #
+      # Gitaly note: JV: to be deprecated in favor of Commit.between
+      def rugged_commits_between(from, to)
         walker = Rugged::Walker.new(rugged)
         walker.sorting(Rugged::SORT_NONE | Rugged::SORT_REVERSE)
 
@@ -708,20 +734,6 @@ module Gitlab
       end
 
       def raw_log(options)
-        default_options = {
-          limit: 10,
-          offset: 0,
-          path: nil,
-          follow: false,
-          skip_merges: false,
-          disable_walk: false,
-          after: nil,
-          before: nil
-        }
-
-        options = default_options.merge(options)
-        options[:limit] ||= 0
-        options[:offset] ||= 0
         actual_ref = options[:ref] || root_ref
         begin
           sha = sha_from_ref(actual_ref)
@@ -857,46 +869,6 @@ module Gitlab
         submodule_data.select { |path, data| data['id'] }
       end
 
-      # Returns true if +commit+ introduced changes to +path+, using commit
-      # trees to make that determination.  Uses the history simplification
-      # rules that `git log` uses by default, where a commit is omitted if it
-      # is TREESAME to any parent.
-      #
-      # If the +follow+ option is true and the file specified by +path+ was
-      # renamed, then the path value is set to the old path.
-      def commit_touches_path?(commit, path, follow, walker)
-        entry = tree_entry(commit, path)
-
-        if commit.parents.empty?
-          # This is the root commit, return true if it has +path+ in its tree
-          return !entry.nil?
-        end
-
-        num_treesame = 0
-        commit.parents.each do |parent|
-          parent_entry = tree_entry(parent, path)
-
-          # Only follow the first TREESAME parent for merge commits
-          if num_treesame > 0
-            walker.hide(parent)
-            next
-          end
-
-          if entry.nil? && parent_entry.nil?
-            num_treesame += 1
-          elsif entry && parent_entry && entry[:oid] == parent_entry[:oid]
-            num_treesame += 1
-          end
-        end
-
-        case num_treesame
-        when 0
-          detect_rename(commit, commit.parents.first, path) if follow
-          true
-        else false
-        end
-      end
-
       # Find the entry for +path+ in the tree for +commit+
       def tree_entry(commit, path)
         pathname = Pathname.new(path)
@@ -924,43 +896,6 @@ module Gitlab
         tmp_entry
       end
 
-      # Compare +commit+ and +parent+ for +path+.  If +path+ is a file and was
-      # renamed in +commit+, then set +path+ to the old filename.
-      def detect_rename(commit, parent, path)
-        diff = parent.diff(commit, paths: [path], disable_pathspec_match: true)
-
-        # If +path+ is a filename, not a directory, then we should only have
-        # one delta.  We don't need to follow renames for directories.
-        return nil if diff.each_delta.count > 1
-
-        delta = diff.each_delta.first
-        if delta.added?
-          full_diff = parent.diff(commit)
-          full_diff.find_similar!
-
-          full_diff.each_delta do |full_delta|
-            if full_delta.renamed? && path == full_delta.new_file[:path]
-              # Look for the old path in ancestors
-              path.replace(full_delta.old_file[:path])
-            end
-          end
-        end
-      end
-
-      # Returns true if the index entry has the special file mode that denotes
-      # a submodule.
-      def submodule?(index_entry)
-        index_entry[:mode] == 57344
-      end
-
-      # Return a Rugged::Index that has read from the tree at +ref_name+
-      def populated_index(ref_name)
-        commit = rev_parse_target(ref_name)
-        index = rugged.index
-        index.read_tree(commit.tree)
-        index
-      end
-
       # Return the Rugged patches for the diff between +from+ and +to+.
       def diff_patches(from, to, options = {}, *paths)
         options ||= {}
@@ -1015,6 +950,14 @@ module Gitlab
         gitaly_ref_client.tags
       end
 
+      def size_by_shelling_out
+        popen(%w(du -sk), path).first.strip.to_i
+      end
+
+      def size_by_gitaly
+        gitaly_repository_client.repository_size
+      end
+
       def count_commits_by_gitaly(options)
         gitaly_commit_client.commit_count(options[:ref], options)
       end
diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb
new file mode 100644
index 00000000000..e28be4b8a38
--- /dev/null
+++ b/lib/gitlab/git/storage.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Git
+    module Storage
+      class Inaccessible < StandardError
+        attr_reader :retry_after
+
+        def initialize(message = nil, retry_after = nil)
+          super(message)
+          @retry_after = retry_after
+        end
+      end
+
+      CircuitOpen = Class.new(Inaccessible)
+
+      REDIS_KEY_PREFIX = 'storage_accessible:'.freeze
+
+      def self.redis
+        Gitlab::Redis::SharedState
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb
new file mode 100644
index 00000000000..9ea9367d4b7
--- /dev/null
+++ b/lib/gitlab/git/storage/circuit_breaker.rb
@@ -0,0 +1,144 @@
+module Gitlab
+  module Git
+    module Storage
+      class CircuitBreaker
+        FailureInfo = Struct.new(:last_failure, :failure_count)
+
+        attr_reader :storage,
+                    :hostname,
+                    :storage_path,
+                    :failure_count_threshold,
+                    :failure_wait_time,
+                    :failure_reset_time,
+                    :storage_timeout
+
+        delegate :last_failure, :failure_count, to: :failure_info
+
+        def self.reset_all!
+          pattern = "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}*"
+
+          Gitlab::Git::Storage.redis.with do |redis|
+            all_storage_keys = redis.keys(pattern)
+            redis.del(*all_storage_keys) unless all_storage_keys.empty?
+          end
+
+          RequestStore.delete(:circuitbreaker_cache)
+        end
+
+        def self.for_storage(storage)
+          cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do
+            Hash.new do |hash, storage_name|
+              hash[storage_name] = new(storage_name)
+            end
+          end
+
+          cached_circuitbreakers[storage]
+        end
+
+        def initialize(storage, hostname = Gitlab::Environment.hostname)
+          @storage = storage
+          @hostname = hostname
+
+          config = Gitlab.config.repositories.storages[@storage]
+          @storage_path = config['path']
+          @failure_count_threshold = config['failure_count_threshold']
+          @failure_wait_time = config['failure_wait_time']
+          @failure_reset_time = config['failure_reset_time']
+          @storage_timeout = config['storage_timeout']
+        end
+
+        def perform
+          return yield unless Feature.enabled?('git_storage_circuit_breaker')
+
+          check_storage_accessible!
+
+          yield
+        end
+
+        def circuit_broken?
+          return false if no_failures?
+
+          recent_failure = last_failure > failure_wait_time.seconds.ago
+          too_many_failures = failure_count > failure_count_threshold
+
+          recent_failure || too_many_failures
+        end
+
+        # Memoizing the `storage_available` call means we only do it once per
+        # request when the storage is available.
+        #
+        # When the storage appears not available, and the memoized value is `false`
+        # we might want to try again.
+        def storage_available?
+          return @storage_available if @storage_available
+
+          if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck
+                                    .storage_available?(storage_path, storage_timeout)
+            track_storage_accessible
+          else
+            track_storage_inaccessible
+          end
+
+          @storage_available
+        end
+
+        def check_storage_accessible!
+          if circuit_broken?
+            raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_wait_time)
+          end
+
+          unless storage_available?
+            raise Gitlab::Git::Storage::Inaccessible.new("#{storage} not accessible", failure_wait_time)
+          end
+        end
+
+        def no_failures?
+          last_failure.blank? && failure_count == 0
+        end
+
+        def track_storage_inaccessible
+          @failure_info = FailureInfo.new(Time.now, failure_count + 1)
+
+          Gitlab::Git::Storage.redis.with do |redis|
+            redis.pipelined do
+              redis.hset(cache_key, :last_failure, last_failure.to_i)
+              redis.hincrby(cache_key, :failure_count, 1)
+              redis.expire(cache_key, failure_reset_time)
+            end
+          end
+        end
+
+        def track_storage_accessible
+          return if no_failures?
+
+          @failure_info = FailureInfo.new(nil, 0)
+
+          Gitlab::Git::Storage.redis.with do |redis|
+            redis.pipelined do
+              redis.hset(cache_key, :last_failure, nil)
+              redis.hset(cache_key, :failure_count, 0)
+            end
+          end
+        end
+
+        def failure_info
+          @failure_info ||= get_failure_info
+        end
+
+        def get_failure_info
+          last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis|
+            redis.hmget(cache_key, :last_failure, :failure_count)
+          end
+
+          last_failure = Time.at(last_failure.to_i) if last_failure.present?
+
+          FailureInfo.new(last_failure, failure_count.to_i)
+        end
+
+        def cache_key
+          @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb
new file mode 100644
index 00000000000..91d8241f17b
--- /dev/null
+++ b/lib/gitlab/git/storage/forked_storage_check.rb
@@ -0,0 +1,55 @@
+module Gitlab
+  module Git
+    module Storage
+      module ForkedStorageCheck
+        extend self
+
+        def storage_available?(path, timeout_seconds = 5)
+          status = timeout_check(path, timeout_seconds)
+
+          status.success?
+        end
+
+        def timeout_check(path, timeout_seconds)
+          filesystem_check_pid = check_filesystem_in_process(path)
+
+          deadline = timeout_seconds.seconds.from_now.utc
+          wait_time = 0.01
+          status = nil
+
+          while status.nil?
+            if deadline > Time.now.utc
+              sleep(wait_time)
+              _pid, status = Process.wait2(filesystem_check_pid, Process::WNOHANG)
+            else
+              Process.kill('KILL', filesystem_check_pid)
+              # Blocking wait, so we are sure the process is gone before continuing
+              _pid, status = Process.wait2(filesystem_check_pid)
+            end
+          end
+
+          status
+        end
+
+        # This will spawn a new 2 processes to do the check:
+        # The outer child (waiter) will spawn another child process (stater).
+        #
+        # The stater is the process is performing the actual filesystem check
+        # the check might hang if the filesystem is acting up.
+        # In this case we will send a `KILL` to the waiter, which will still
+        # be responsive while the stater is hanging.
+        def check_filesystem_in_process(path)
+          spawn('ruby', '-e', ruby_check, path, [:out, :err] => '/dev/null')
+        end
+
+        def ruby_check
+          <<~RUBY_FILESYSTEM_CHECK
+          inner_pid = fork { File.stat(ARGV.first) }
+          Process.waitpid(inner_pid)
+          exit $?.exitstatus
+          RUBY_FILESYSTEM_CHECK
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb
new file mode 100644
index 00000000000..2d723147f4f
--- /dev/null
+++ b/lib/gitlab/git/storage/health.rb
@@ -0,0 +1,91 @@
+module Gitlab
+  module Git
+    module Storage
+      class Health
+        attr_reader :storage_name, :info
+
+        def self.pattern_for_storage(storage_name)
+          "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage_name}:*"
+        end
+
+        def self.for_all_storages
+          storage_names = Gitlab.config.repositories.storages.keys
+          results_per_storage = nil
+
+          Gitlab::Git::Storage.redis.with do |redis|
+            keys_per_storage = all_keys_for_storages(storage_names, redis)
+            results_per_storage = load_for_keys(keys_per_storage, redis)
+          end
+
+          results_per_storage.map do |name, info|
+            info.each { |i| i[:failure_count] = i[:failure_count].value.to_i }
+            new(name, info)
+          end
+        end
+
+        def self.all_keys_for_storages(storage_names, redis)
+          keys_per_storage = {}
+
+          redis.pipelined do
+            storage_names.each do |storage_name|
+              pattern = pattern_for_storage(storage_name)
+
+              keys_per_storage[storage_name] = redis.keys(pattern)
+            end
+          end
+
+          keys_per_storage
+        end
+
+        def self.load_for_keys(keys_per_storage, redis)
+          info_for_keys = {}
+
+          redis.pipelined do
+            keys_per_storage.each do |storage_name, keys_future|
+              info_for_storage = keys_future.value.map do |key|
+                { name: key, failure_count: redis.hget(key, :failure_count) }
+              end
+
+              info_for_keys[storage_name] = info_for_storage
+            end
+          end
+
+          info_for_keys
+        end
+
+        def self.for_failing_storages
+          for_all_storages.select(&:failing?)
+        end
+
+        def initialize(storage_name, info)
+          @storage_name = storage_name
+          @info = info
+        end
+
+        def failing_info
+          @failing_info ||= info.select { |info_for_host| info_for_host[:failure_count] > 0 }
+        end
+
+        def failing?
+          failing_info.any?
+        end
+
+        def failing_on_hosts
+          @failing_on_hosts ||= failing_info.map do |info_for_host|
+            info_for_host[:name].split(':').last
+          end
+        end
+
+        def failing_circuit_breakers
+          @failing_circuit_breakers ||= failing_on_hosts.map do |hostname|
+            CircuitBreaker.new(storage_name, hostname)
+          end
+        end
+
+        def total_failures
+          @total_failures ||= failing_info.sum { |info_for_host| info_for_host[:failure_count] }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index c90ef282fdd..70177cd0fec 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -100,5 +100,9 @@ module Gitlab
       path = Rails.root.join(SERVER_VERSION_FILE)
       path.read.chomp
     end
+
+    def self.encode(s)
+      s.dup.force_encoding(Encoding::ASCII_8BIT)
+    end
   end
 end
diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb
deleted file mode 100644
index 61fe462d762..00000000000
--- a/lib/gitlab/gitaly_client/commit.rb
+++ /dev/null
@@ -1,14 +0,0 @@
-module Gitlab
-  module GitalyClient
-    class Commit
-      attr_reader :repository, :gitaly_commit
-
-      delegate :id, :subject, :body, :author, :committer, :parent_ids, to: :gitaly_commit
-
-      def initialize(repository, gitaly_commit)
-        @repository = repository
-        @gitaly_commit = gitaly_commit
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index ac6817e6d0e..692d7e02eef 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -29,22 +29,21 @@ module Gitlab
 
         request = Gitaly::CommitDiffRequest.new(request_params)
         response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request)
-        Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options.merge(from_gitaly: true))
+        GitalyClient::DiffStitcher.new(response)
       end
 
       def commit_deltas(commit)
         request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit))
         response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request)
-        response.flat_map do |msg|
-          msg.deltas.map { |d| Gitlab::Git::Diff.new(d) }
-        end
+
+        response.flat_map { |msg| msg.deltas }
       end
 
       def tree_entry(ref, path, limit = nil)
         request = Gitaly::TreeEntryRequest.new(
           repository: @gitaly_repo,
           revision: ref,
-          path: path.dup.force_encoding(Encoding::ASCII_8BIT),
+          path: GitalyClient.encode(path),
           limit: limit.to_i
         )
 
@@ -100,15 +99,14 @@ module Gitlab
       def last_commit_for_path(revision, path)
         request = Gitaly::LastCommitForPathRequest.new(
           repository: @gitaly_repo,
-          revision: revision.force_encoding(Encoding::ASCII_8BIT),
-          path: path.to_s.force_encoding(Encoding::ASCII_8BIT)
+          revision: GitalyClient.encode(revision),
+          path: GitalyClient.encode(path.to_s)
         )
 
         gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request).commit
         return unless gitaly_commit
 
-        commit = GitalyClient::Commit.new(@repository, gitaly_commit)
-        Gitlab::Git::Commit.new(commit)
+        Gitlab::Git::Commit.new(@repository, gitaly_commit)
       end
 
       def between(from, to)
@@ -135,6 +133,20 @@ module Gitlab
         consume_commits_response(response)
       end
 
+      def commits_by_message(query, revision: '', path: '', limit: 1000, offset: 0)
+        request = Gitaly::CommitsByMessageRequest.new(
+          repository: @gitaly_repo,
+          query: query,
+          revision: revision.to_s.force_encoding(Encoding::ASCII_8BIT),
+          path: path.to_s.force_encoding(Encoding::ASCII_8BIT),
+          limit: limit.to_i,
+          offset: offset.to_i
+        )
+
+        response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request)
+        consume_commits_response(response)
+      end
+
       def languages(ref = nil)
         request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '')
         response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request)
@@ -153,10 +165,21 @@ module Gitlab
         response.reduce("") { |memo, msg| memo << msg.data }
       end
 
+      def find_commit(revision)
+        request = Gitaly::FindCommitRequest.new(
+          repository: @gitaly_repo,
+          revision: GitalyClient.encode(revision)
+        )
+
+        response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request)
+
+        response.commit
+      end
+
       private
 
       def commit_diff_request_params(commit, options = {})
-        parent_id = commit.parents[0]&.id || EMPTY_TREE_ID
+        parent_id = commit.parent_ids.first || EMPTY_TREE_ID
 
         {
           repository: @gitaly_repo,
@@ -169,8 +192,7 @@ module Gitlab
       def consume_commits_response(response)
         response.flat_map do |message|
           message.commits.map do |gitaly_commit|
-            commit = GitalyClient::Commit.new(@repository, gitaly_commit)
-            Gitlab::Git::Commit.new(commit)
+            Gitlab::Git::Commit.new(@repository, gitaly_commit)
           end
         end
       end
diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb
index b0f7548b7dc..919fb68b8c7 100644
--- a/lib/gitlab/gitaly_client/ref_service.rb
+++ b/lib/gitlab/gitaly_client/ref_service.rb
@@ -16,8 +16,7 @@ module Gitlab
 
         response.flat_map do |message|
           message.branches.map do |branch|
-            gitaly_commit = GitalyClient::Commit.new(@repository, branch.target)
-            target_commit = Gitlab::Git::Commit.decorate(gitaly_commit)
+            target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target)
             Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit)
           end
         end
@@ -102,8 +101,7 @@ module Gitlab
         response.flat_map do |message|
           message.tags.map do |gitaly_tag|
             if gitaly_tag.target_commit.present?
-              commit = GitalyClient::Commit.new(@repository, gitaly_tag.target_commit)
-              gitaly_commit = Gitlab::Git::Commit.new(commit)
+              gitaly_commit = Gitlab::Git::Commit.decorate(@repository, gitaly_tag.target_commit)
             end
 
             Gitlab::Git::Tag.new(
@@ -141,7 +139,7 @@ module Gitlab
           committer_email: response.commit_committer.email.dup
         }
 
-        Gitlab::Git::Commit.decorate(hash)
+        Gitlab::Git::Commit.decorate(@repository, hash)
       end
     end
   end
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index 13e75b256a7..79ce784f2f2 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -27,6 +27,11 @@ module Gitlab
         request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo)
         GitalyClient.call(@storage, :repository_service, :repack_incremental, request)
       end
+
+      def repository_size
+        request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo)
+        GitalyClient.call(@storage, :repository_service, :repository_size, request).size
+      end
     end
   end
 end
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb
index 9e91c135956..eef97f54962 100644
--- a/lib/gitlab/health_checks/fs_shards_check.rb
+++ b/lib/gitlab/health_checks/fs_shards_check.rb
@@ -10,7 +10,9 @@ module Gitlab
         def readiness
           repository_storages.map do |storage_name|
             begin
-              if !storage_stat_test(storage_name)
+              if !storage_circuitbreaker_test(storage_name)
+                HealthChecks::Result.new(false, 'circuitbreaker tripped', shard: storage_name)
+              elsif !storage_stat_test(storage_name)
                 HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name)
               else
                 with_temp_file(storage_name) do |tmp_file_path|
@@ -36,7 +38,8 @@ module Gitlab
             [
               storage_stat_metrics(storage_name),
               storage_write_metrics(storage_name),
-              storage_read_metrics(storage_name)
+              storage_read_metrics(storage_name),
+              storage_circuitbreaker_metrics(storage_name)
             ].flatten
           end
         end
@@ -121,6 +124,12 @@ module Gitlab
           file_contents == RANDOM_STRING
         end
 
+        def storage_circuitbreaker_test(storage_name)
+          Gitlab::Git::Storage::CircuitBreaker.new(storage_name).perform { "OK" }
+        rescue Gitlab::Git::Storage::Inaccessible
+          nil
+        end
+
         def storage_stat_metrics(storage_name)
           operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, shard: storage_name) do
             with_timing { storage_stat_test(storage_name) }
@@ -143,6 +152,14 @@ module Gitlab
             end
           end
         end
+
+        def storage_circuitbreaker_metrics(storage_name)
+          operation_metrics(:filesystem_circuitbreaker,
+                            :filesystem_circuitbreaker_latency_seconds,
+                            shard: storage_name) do
+            with_timing { storage_circuitbreaker_test(storage_name) }
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
index cc282d1415b..5d106b5c075 100644
--- a/lib/gitlab/i18n.rb
+++ b/lib/gitlab/i18n.rb
@@ -16,7 +16,8 @@ module Gitlab
       'eo' => 'Esperanto',
       'it' => 'Italiano',
       'uk' => 'Українська',
-      'ja' => '日本語'
+      'ja' => '日本語',
+      'ko' => '한국어'
     }.freeze
 
     def available_locales
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index c8ad3a7a5e0..c5c05bfe2fb 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -101,6 +101,7 @@ excluded_attributes:
   merge_requests:
     - :milestone_id
     - :ref_fetched
+    - :merge_jid
   award_emoji:
     - :awardable_id
   statuses:
diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb
index 52276cbcd9a..5404dc11a87 100644
--- a/lib/gitlab/import_sources.rb
+++ b/lib/gitlab/import_sources.rb
@@ -8,7 +8,7 @@ module Gitlab
     ImportSource = Struct.new(:name, :title, :importer)
 
     ImportTable = [
-      ImportSource.new('github',         'GitHub',        Gitlab::GithubImport::Importer),
+      ImportSource.new('github',         'GitHub',        Github::Import),
       ImportSource.new('bitbucket',      'Bitbucket',     Gitlab::BitbucketImport::Importer),
       ImportSource.new('gitlab',         'GitLab.com',    Gitlab::GitlabImport::Importer),
       ImportSource.new('google_code',    'Google Code',   Gitlab::GoogleCodeImport::Importer),
diff --git a/lib/gitlab/key_fingerprint.rb b/lib/gitlab/key_fingerprint.rb
index b75ae512d92..d9a79f7c291 100644
--- a/lib/gitlab/key_fingerprint.rb
+++ b/lib/gitlab/key_fingerprint.rb
@@ -1,55 +1,48 @@
 module Gitlab
   class KeyFingerprint
-    include Gitlab::Popen
+    attr_reader :key, :ssh_key
 
-    attr_accessor :key
+    # Unqualified MD5 fingerprint for compatibility
+    delegate :fingerprint, to: :ssh_key, allow_nil: true
 
     def initialize(key)
       @key = key
-    end
-
-    def fingerprint
-      cmd_status = 0
-      cmd_output = ''
-
-      Tempfile.open('gitlab_key_file') do |file|
-        file.puts key
-        file.rewind
-
-        cmd = []
-        cmd.push('ssh-keygen')
-        cmd.push('-E', 'md5') if explicit_fingerprint_algorithm?
-        cmd.push('-lf', file.path)
-
-        cmd_output, cmd_status = popen(cmd, '/tmp')
-      end
-
-      return nil unless cmd_status.zero?
 
-      # 16 hex bytes separated by ':', optionally starting with "MD5:"
-      fingerprint_matches = cmd_output.match(/(MD5:)?(?<fingerprint>(\h{2}:){15}\h{2})/)
-      return nil unless fingerprint_matches
-
-      fingerprint_matches[:fingerprint]
+      @ssh_key =
+        begin
+          Net::SSH::KeyFactory.load_data_public_key(key)
+        rescue Net::SSH::Exception, NotImplementedError
+        end
     end
 
-    private
-
-    def explicit_fingerprint_algorithm?
-      # OpenSSH 6.8 introduces a new default output format for fingerprints.
-      # Check the version and decide which command to use.
-
-      version_output, version_status = popen(%w(ssh -V))
-      return false unless version_status.zero?
+    def valid?
+      ssh_key.present?
+    end
 
-      version_matches = version_output.match(/OpenSSH_(?<major>\d+)\.(?<minor>\d+)/)
-      return false unless version_matches
+    def type
+      return unless valid?
 
-      version_info = Gitlab::VersionInfo.new(version_matches[:major].to_i, version_matches[:minor].to_i)
+      parts = ssh_key.ssh_type.split('-')
+      parts.shift if parts[0] == 'ssh'
 
-      required_version_info = Gitlab::VersionInfo.new(6, 8)
+      parts[0].upcase
+    end
 
-      version_info >= required_version_info 
+    def bits
+      return unless valid?
+
+      case type
+      when 'RSA'
+        ssh_key.n.num_bits
+      when 'DSS', 'DSA'
+        ssh_key.p.num_bits
+      when 'ECDSA'
+        ssh_key.group.order.num_bits
+      when 'ED25519'
+        256
+      else
+        raise "Unsupported key type: #{type}"
+      end
     end
   end
 end
diff --git a/lib/gitlab/metrics/base_sampler.rb b/lib/gitlab/metrics/base_sampler.rb
index 219accfc029..716d20bb91a 100644
--- a/lib/gitlab/metrics/base_sampler.rb
+++ b/lib/gitlab/metrics/base_sampler.rb
@@ -1,20 +1,7 @@
 require 'logger'
 module Gitlab
   module Metrics
-    class BaseSampler
-      def self.initialize_instance(*args)
-        raise "#{name} singleton instance already initialized" if @instance
-        @instance = new(*args)
-        at_exit(&@instance.method(:stop))
-        @instance
-      end
-
-      def self.instance
-        @instance
-      end
-
-      attr_reader :running
-
+    class BaseSampler < Daemon
       # interval - The sampling interval in seconds.
       def initialize(interval)
         interval_half = interval.to_f / 2
@@ -22,44 +9,7 @@ module Gitlab
         @interval = interval
         @interval_steps = (-interval_half..interval_half).step(0.1).to_a
 
-        @mutex = Mutex.new
-      end
-
-      def enabled?
-        true
-      end
-
-      def start
-        return unless enabled?
-
-        @mutex.synchronize do
-          return if running
-          @running = true
-
-          @thread = Thread.new do
-            sleep(sleep_interval)
-
-            while running
-              safe_sample
-
-              sleep(sleep_interval)
-            end
-          end
-        end
-      end
-
-      def stop
-        @mutex.synchronize do
-          return unless running
-
-          @running = false
-
-          if @thread
-            @thread.wakeup if @thread.alive?
-            @thread.join
-            @thread = nil
-          end
-        end
+        super()
       end
 
       def safe_sample
@@ -81,7 +31,7 @@ module Gitlab
       #    potentially missing anything that happens in between samples).
       # 2. Don't sample data at the same interval two times in a row.
       def sleep_interval
-        while step = @interval_steps.sample
+        while (step = @interval_steps.sample)
           if step != @last_step
             @last_step = step
 
@@ -89,6 +39,25 @@ module Gitlab
           end
         end
       end
+
+      private
+
+      attr_reader :running
+
+      def start_working
+        @running = true
+        sleep(sleep_interval)
+
+        while running
+          safe_sample
+
+          sleep(sleep_interval)
+        end
+      end
+
+      def stop_working
+        @running = false
+      end
     end
   end
 end
diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb
new file mode 100644
index 00000000000..5980a4ded2b
--- /dev/null
+++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb
@@ -0,0 +1,39 @@
+require 'webrick'
+require 'prometheus/client/rack/exporter'
+
+module Gitlab
+  module Metrics
+    class SidekiqMetricsExporter < Daemon
+      def enabled?
+        Gitlab::Metrics.metrics_folder_present? && settings.enabled
+      end
+
+      def settings
+        Settings.monitoring.sidekiq_exporter
+      end
+
+      private
+
+      attr_reader :server
+
+      def start_working
+        @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address)
+        server.mount "/", Rack::Handler::WEBrick, rack_app
+        server.start
+      end
+
+      def stop_working
+        server.shutdown
+        @server = nil
+      end
+
+      def rack_app
+        Rack::Builder.app do
+          use Rack::Deflater
+          use ::Prometheus::Client::Rack::Exporter
+          run -> (env) { [404, {}, ['']] }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb
new file mode 100644
index 00000000000..cf461adf697
--- /dev/null
+++ b/lib/gitlab/project_template.rb
@@ -0,0 +1,45 @@
+module Gitlab
+  class ProjectTemplate
+    attr_reader :title, :name
+
+    def initialize(name, title)
+      @name, @title = name, title
+    end
+
+    alias_method :logo, :name
+
+    def file
+      archive_path.open
+    end
+
+    def archive_path
+      Rails.root.join("vendor/project_templates/#{name}.tar.gz")
+    end
+
+    def clone_url
+      "https://gitlab.com/gitlab-org/project-templates/#{name}.git"
+    end
+
+    def ==(other)
+      name == other.name && title == other.title
+    end
+
+    TEMPLATES_TABLE = [
+      ProjectTemplate.new('rails', 'Ruby on Rails')
+    ].freeze
+
+    class << self
+      def all
+        TEMPLATES_TABLE
+      end
+
+      def find(name)
+        all.find { |template| template.name == name.to_s }
+      end
+
+      def archive_directory
+        Rails.root.join("vendor_directory/project_templates")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
index db4708b22e4..32fe8201a8d 100644
--- a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
+++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
@@ -5,7 +5,7 @@ module Gitlab
         include QueryAdditionalMetrics
 
         def query(environment_id)
-          Environment.find_by(id: environment_id).try do |environment|
+          ::Environment.find_by(id: environment_id).try do |environment|
             query_metrics(
               common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f)
             )
diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb
index 66f29d95177..1d17d3cfd56 100644
--- a/lib/gitlab/prometheus/queries/environment_query.rb
+++ b/lib/gitlab/prometheus/queries/environment_query.rb
@@ -3,7 +3,7 @@ module Gitlab
     module Queries
       class EnvironmentQuery < BaseQuery
         def query(environment_id)
-          Environment.find_by(id: environment_id).try do |environment|
+          ::Environment.find_by(id: environment_id).try do |environment|
             environment_slug = environment.slug
             timeframe_start = 8.hours.ago.to_f
             timeframe_end = Time.now.to_f
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index 4366ff336ef..0cb28732402 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -105,12 +105,24 @@ module Gitlab
     #   fetch_remote("gitlab/gitlab-ci", "upstream")
     #
     # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
-    def fetch_remote(storage, name, remote, forced: false, no_tags: false)
+    def fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false)
       args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"]
       args << '--force' if forced
       args << '--no-tags' if no_tags
 
-      gitlab_shell_fast_execute_raise_error(args)
+      vars = {}
+
+      if ssh_auth&.ssh_import?
+        if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
+          vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key
+        end
+
+        if ssh_auth.ssh_known_hosts.present?
+          vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts
+        end
+      end
+
+      gitlab_shell_fast_execute_raise_error(args, vars)
     end
 
     # Move repository
@@ -293,15 +305,15 @@ module Gitlab
       false
     end
 
-    def gitlab_shell_fast_execute_raise_error(cmd)
-      output, status = gitlab_shell_fast_execute_helper(cmd)
+    def gitlab_shell_fast_execute_raise_error(cmd, vars = {})
+      output, status = gitlab_shell_fast_execute_helper(cmd, vars)
 
       raise Error, output unless status.zero?
       true
     end
 
-    def gitlab_shell_fast_execute_helper(cmd)
-      vars = ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS)
+    def gitlab_shell_fast_execute_helper(cmd, vars = {})
+      vars.merge!(ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS))
 
       # Don't pass along the entire parent environment to prevent gitlab-shell
       # from wasting I/O by searching through GEM_PATH
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index e0ac21305a5..748e0a29184 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -27,8 +27,8 @@ module Gitlab
             ci_pipeline_schedules: ::Ci::PipelineSchedule.count,
             deploy_keys: DeployKey.count,
             deployments: Deployment.count,
-            environments: Environment.count,
-            in_review_folder: Environment.in_review_folder.count,
+            environments: ::Environment.count,
+            in_review_folder: ::Environment.in_review_folder.count,
             groups: Group.count,
             issues: Issue.count,
             keys: Key.count,
diff --git a/lib/haml_lint/inline_javascript.rb b/lib/haml_lint/inline_javascript.rb
new file mode 100644
index 00000000000..05668c69006
--- /dev/null
+++ b/lib/haml_lint/inline_javascript.rb
@@ -0,0 +1,16 @@
+unless Rails.env.production?
+  require 'haml_lint/haml_visitor'
+  require 'haml_lint/linter'
+  require 'haml_lint/linter_registry'
+
+  module HamlLint
+    class Linter::InlineJavaScript < Linter
+      include LinterRegistry
+
+      def visit_filter(node)
+        return unless node.filter_type == 'javascript'
+        record_lint(node, 'Inline JavaScript is discouraged (https://docs.gitlab.com/ee/development/gotchas.html#do-not-use-inline-javascript-in-views)')
+      end
+    end
+  end
+end
diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb
index 688a79c0441..ef08bd46e17 100644
--- a/lib/mattermost/session.rb
+++ b/lib/mattermost/session.rb
@@ -36,11 +36,12 @@ module Mattermost
 
     def with_session
       with_lease do
-        raise Mattermost::NoSessionError unless create
+        create
 
         begin
           yield self
-        rescue Errno::ECONNREFUSED
+        rescue Errno::ECONNREFUSED => e
+          Rails.logger.error(e.message + "\n" + e.backtrace.join("\n"))
           raise Mattermost::NoSessionError
         ensure
           destroy
@@ -85,10 +86,12 @@ module Mattermost
     private
 
     def create
-      return unless oauth_uri
-      return unless token_uri
+      raise Mattermost::NoSessionError unless oauth_uri
+      raise Mattermost::NoSessionError unless token_uri
 
       @token = request_token
+      raise Mattermost::NoSessionError unless @token
+
       @headers = {
         Authorization: "Bearer #{@token}"
       }
@@ -106,11 +109,16 @@ module Mattermost
       @oauth_uri = nil
 
       response = get("/api/v3/oauth/gitlab/login", follow_redirects: false)
-      return unless 300 <= response.code && response.code < 400
+      return unless (300...400) === response.code
 
       redirect_uri = response.headers['location']
       return unless redirect_uri
 
+      oauth_cookie = parse_cookie(response)
+      @headers = {
+        Cookie: oauth_cookie.to_cookie_string
+      }
+
       @oauth_uri = URI.parse(redirect_uri)
     end
 
@@ -124,7 +132,7 @@ module Mattermost
     def request_token
       response = get(token_uri, follow_redirects: false)
 
-      if 200 <= response.code && response.code < 400
+      if (200...400) === response.code
         response.headers['token']
       end
     end
@@ -156,5 +164,11 @@ module Mattermost
     rescue Errno::ECONNREFUSED => e
       raise Mattermost::ConnectionError.new(e.message)
     end
+
+    def parse_cookie(response)
+      cookie_hash = CookieHash.new
+      response.get_fields('Set-Cookie').each { |c| cookie_hash.add_cookies(c) }
+      cookie_hash
+    end
   end
 end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index 3703f9cfb5c..aaf00bd703a 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -21,7 +21,7 @@ namespace :gitlab do
         create_gitaly_configuration
         # In CI we run scripts/gitaly-test-build instead of this command
         unless ENV['CI'].present?
-          Bundler.with_original_env { run_command!([command]) }
+          Bundler.with_original_env { run_command!(%w[/usr/bin/env -u RUBYOPT] + [command]) }
         end
       end
     end
@@ -66,6 +66,7 @@ namespace :gitlab do
       config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages }
       config[:auth] = { token: 'secret' } if Rails.env.test?
       config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby
+      config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
       TOML.dump(config)
     end
 
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
index 59c32bbe7a4..a7e30423c7a 100644
--- a/lib/tasks/gitlab/update_templates.rake
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -4,6 +4,55 @@ namespace :gitlab do
     TEMPLATE_DATA.each { |template| update(template) }
   end
 
+  desc "GitLab | Update project templates"
+  task :update_project_templates do
+    if Rails.env.production?
+      puts "This rake task is not meant fo production instances".red
+      exit(1)
+    end
+    admin = User.find_by(admin: true)
+
+    unless admin
+      puts "No admin user could be found".red
+      exit(1)
+    end
+
+    Gitlab::ProjectTemplate.all.each do |template|
+      params = {
+        import_url: template.clone_url,
+        namespace_id: admin.namespace.id,
+        path: template.title,
+        skip_wiki: true
+      }
+
+      puts "Creating project for #{template.name}"
+      project = Projects::CreateService.new(admin, params).execute
+
+      loop do
+        if project.finished?
+          puts "Import finished for #{template.name}"
+          break
+        end
+
+        if project.failed?
+          puts "Failed to import from #{project_params[:import_url]}".red
+          exit(1)
+        end
+
+        puts "Waiting for the import to finish"
+
+        sleep(5)
+        project.reload
+      end
+
+      Projects::ImportExport::ExportService.new(project, admin).execute
+      FileUtils.cp(project.export_project_path, template.archive_path)
+      Projects::DestroyService.new(admin, project).execute
+      puts "Exported #{template.name}".green
+    end
+    puts "Done".green
+  end
+
   def update(template)
     sub_dir = template.repo_url.match(/([A-Za-z-]+)\.git\z/)[1]
     dir = File.join(vendor_directory, sub_dir)
diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake
index 609dfaa48e3..ad2d034b0b4 100644
--- a/lib/tasks/haml-lint.rake
+++ b/lib/tasks/haml-lint.rake
@@ -1,5 +1,6 @@
 unless Rails.env.production?
   require 'haml_lint/rake_task'
+  require 'haml_lint/inline_javascript'
 
   HamlLint::RakeTask.new
 end
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index 50b8e331469..96b8f59242c 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -7,7 +7,7 @@ class GithubImport
   end
 
   def initialize(token, gitlab_username, project_path, extras)
-    @options = { url: 'https://api.github.com', token: token, verbose: true }
+    @options = { token: token, verbose: true }
     @project_path = project_path
     @current_user = User.find_by_username(gitlab_username)
     @github_repo = extras.empty? ? nil : extras.first
@@ -62,6 +62,7 @@ class GithubImport
         visibility_level: visibility_level,
         import_type: 'github',
         import_source: @repo['full_name'],
+        import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@"),
         skip_wiki: @repo['has_wiki']
       ).execute
     end