Merge branch 'master' into ide

* master: (177 commits)
  Add changelog
  Bump gitlab-shell version to 5.8.0 to fix Git for Windows 2.14
  Make contextual sidebar collapsible
  Fixed sidebar context header hover colors
  Use correct `Environment`-class within `Gitlab` namespace
  Remove gl.Activities from Commits page
  Move `let` calls inside the `describe` block using them
  Add `/assign me` alias support for assigning issuables to oneself
  GRPC::Unavailable (< GRPC::BadStatus) is wrapped in a CommandError
  Use `broken_storage` in the fs_shards_spec.
  Eager load project creators for project dashboards
  Memoize a user's personal projects count
  Remove redundant query from User#recent_push
  Improve checking if projects would be returned
  Change spelling of gitlab-shell
  Remove unused #tree-holder
  Add custom linter for inline JavaScript to haml_lint
  Rename user_can_admin? because it's more accurate
  Synchronous zanata community contribution translation
  Add Korean translation to i18n
  ...

30 files changed, 692 insertions, 152 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 982a2b88d62..94df543853b 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -95,6 +95,7 @@ module API
     mount ::API::Boards
     mount ::API::Branches
     mount ::API::BroadcastMessages
+    mount ::API::CircuitBreakers
     mount ::API::Commits
     mount ::API::CommitStatuses
     mount ::API::DeployKeys
diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb
new file mode 100644
index 00000000000..118883f5ea5
--- /dev/null
+++ b/lib/api/circuit_breakers.rb
@@ -0,0 +1,50 @@
+module API
+  class CircuitBreakers < Grape::API
+    before { authenticated_as_admin! }
+
+    resource :circuit_breakers do
+      params do
+        requires :type,
+                 type: String,
+                 desc: "The type of circuitbreaker",
+                 values: ['repository_storage']
+      end
+      resource ':type' do
+        namespace '', requirements: { type: 'repository_storage' } do
+          helpers do
+            def failing_storage_health
+              @failing_storage_health ||= Gitlab::Git::Storage::Health.for_failing_storages
+            end
+
+            def storage_health
+              @failing_storage_health ||= Gitlab::Git::Storage::Health.for_all_storages
+            end
+          end
+
+          desc 'Get all failing git storages' do
+            detail 'This feature was introduced in GitLab 9.5'
+            success Entities::RepositoryStorageHealth
+          end
+          get do
+            present storage_health, with: Entities::RepositoryStorageHealth
+          end
+
+          desc 'Get all failing git storages' do
+            detail 'This feature was introduced in GitLab 9.5'
+            success Entities::RepositoryStorageHealth
+          end
+          get 'failing' do
+            present failing_storage_health, with: Entities::RepositoryStorageHealth
+          end
+
+          desc 'Reset all storage failures and open circuitbreaker' do
+            detail 'This feature was introduced in GitLab 9.5'
+          end
+          delete do
+            Gitlab::Git::Storage::CircuitBreaker.reset_all!
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 298831a8fdb..94b438db499 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -66,13 +66,6 @@ module API
       expose :job_events
     end
 
-    class BasicProjectDetails < Grape::Entity
-      expose :id
-      expose :http_url_to_repo, :web_url
-      expose :name, :name_with_namespace
-      expose :path, :path_with_namespace
-    end
-
     class SharedGroup < Grape::Entity
       expose :group_id
       expose :group_name do |group_link, options|
@@ -81,7 +74,16 @@ module API
       expose :group_access, as: :group_access_level
     end
 
-    class Project < Grape::Entity
+    class BasicProjectDetails < Grape::Entity
+      expose :id, :description, :default_branch, :tag_list
+      expose :ssh_url_to_repo, :http_url_to_repo, :web_url
+      expose :name, :name_with_namespace
+      expose :path, :path_with_namespace
+      expose :star_count, :forks_count
+      expose :created_at, :last_activity_at
+    end
+
+    class Project < BasicProjectDetails 
       include ::API::Helpers::RelatedResourcesHelpers
 
       expose :_links do
@@ -114,12 +116,9 @@ module API
         end
       end
 
-      expose :id, :description, :default_branch, :tag_list
       expose :archived?, as: :archived
-      expose :visibility, :ssh_url_to_repo, :http_url_to_repo, :web_url
+      expose :visibility
       expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group }
-      expose :name, :name_with_namespace
-      expose :path, :path_with_namespace
       expose :container_registry_enabled
 
       # Expose old field names with the new permissions methods to keep API compatible
@@ -129,7 +128,6 @@ module API
       expose(:jobs_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) }
       expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) }
 
-      expose :created_at, :last_activity_at
       expose :shared_runners_enabled
       expose :lfs_enabled?, as: :lfs_enabled
       expose :creator_id
@@ -140,7 +138,6 @@ module API
       expose :avatar_url do |user, options|
         user.avatar_url(only_path: false)
       end
-      expose :star_count, :forks_count
       expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) }
       expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
       expose :public_builds, as: :public_jobs
@@ -954,5 +951,11 @@ module API
       expose :ip_address
       expose :submitted, as: :akismet_submitted
     end
+
+    class RepositoryStorageHealth < Grape::Entity
+      expose :storage_name
+      expose :failing_on_hosts
+      expose :total_failures
+    end
   end
 end
diff --git a/lib/api/todos.rb b/lib/api/todos.rb
index d1f7e364029..55191169dd4 100644
--- a/lib/api/todos.rb
+++ b/lib/api/todos.rb
@@ -59,10 +59,10 @@ module API
         requires :id, type: Integer, desc: 'The ID of the todo being marked as done'
       end
       post ':id/mark_as_done' do
-        todo = current_user.todos.find(params[:id])
-        TodoService.new.mark_todos_as_done([todo], current_user)
+        TodoService.new.mark_todos_as_done_by_ids(params[:id], current_user)
+        todo = Todo.find(params[:id])
 
-        present todo.reload, with: Entities::Todo, current_user: current_user
+        present todo, with: Entities::Todo, current_user: current_user
       end
 
       desc 'Mark all todos as done'
diff --git a/lib/api/v3/todos.rb b/lib/api/v3/todos.rb
index e3b311d61cd..2f2cf259987 100644
--- a/lib/api/v3/todos.rb
+++ b/lib/api/v3/todos.rb
@@ -11,10 +11,10 @@ module API
           requires :id, type: Integer, desc: 'The ID of the todo being marked as done'
         end
         delete ':id' do
-          todo = current_user.todos.find(params[:id])
-          TodoService.new.mark_todos_as_done([todo], current_user)
+          TodoService.new.mark_todos_as_done_by_ids(params[:id], current_user)
+          todo = Todo.find(params[:id])
 
-          present todo.reload, with: ::API::Entities::Todo, current_user: current_user
+          present todo, with: ::API::Entities::Todo, current_user: current_user
         end
 
         desc 'Mark all todos as done'
diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb
index c7801cb5baf..ad08c0905e2 100644
--- a/lib/banzai/renderer.rb
+++ b/lib/banzai/renderer.rb
@@ -132,6 +132,8 @@ module Banzai
     end
 
     def self.cacheless_render(text, context = {})
+      return text.to_s unless text.present?
+
       Gitlab::Metrics.measure(:banzai_cacheless_render) do
         result = render_result(text, context)
 
diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb
index b1eb1a6cef1..ae65653645b 100644
--- a/lib/declarative_policy.rb
+++ b/lib/declarative_policy.rb
@@ -28,7 +28,13 @@ module DeclarativePolicy
 
       subject = find_delegate(subject)
 
-      class_for_class(subject.class)
+      policy_class = class_for_class(subject.class)
+      raise "no policy for #{subject.class.name}" if policy_class.nil?
+      policy_class
+    end
+
+    def has_policy?(subject)
+      !class_for_class(subject.class).nil?
     end
 
     private
@@ -51,9 +57,7 @@ module DeclarativePolicy
         end
       end
 
-      policy_class = subject_class.instance_variable_get(CLASS_CACHE_IVAR)
-      raise "no policy for #{subject.class.name}" if policy_class.nil?
-      policy_class
+      subject_class.instance_variable_get(CLASS_CACHE_IVAR)
     end
 
     def compute_class_for_class(subject_class)
@@ -71,6 +75,8 @@ module DeclarativePolicy
           nil
         end
       end
+
+      nil
     end
 
     def find_delegate(subject)
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 9bed81e7327..7d3aa532750 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -218,7 +218,8 @@ module Gitlab
       def full_authentication_abilities
         read_authentication_abilities + [
           :push_code,
-          :create_container_image
+          :create_container_image,
+          :admin_container_image
         ]
       end
       alias_method :api_scope_authentication_abilities, :full_authentication_abilities
diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb
new file mode 100644
index 00000000000..0fbc6b70989
--- /dev/null
+++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb
@@ -0,0 +1,107 @@
+module Gitlab
+  module BackgroundMigration
+    class DeserializeMergeRequestDiffsAndCommits
+      attr_reader :diff_ids, :commit_rows, :file_rows
+
+      class MergeRequestDiff < ActiveRecord::Base
+        self.table_name = 'merge_request_diffs'
+      end
+
+      BUFFER_ROWS = 1000
+
+      def perform(start_id, stop_id)
+        merge_request_diffs = MergeRequestDiff
+                               .select(:id, :st_commits, :st_diffs)
+                               .where('st_commits IS NOT NULL OR st_diffs IS NOT NULL')
+                               .where(id: start_id..stop_id)
+
+        reset_buffers!
+
+        merge_request_diffs.each do |merge_request_diff|
+          commits, files = single_diff_rows(merge_request_diff)
+
+          diff_ids << merge_request_diff.id
+          commit_rows.concat(commits)
+          file_rows.concat(files)
+
+          if diff_ids.length > BUFFER_ROWS ||
+              commit_rows.length > BUFFER_ROWS ||
+              file_rows.length > BUFFER_ROWS
+
+            flush_buffers!
+          end
+        end
+
+        flush_buffers!
+      end
+
+      private
+
+      def reset_buffers!
+        @diff_ids = []
+        @commit_rows = []
+        @file_rows = []
+      end
+
+      def flush_buffers!
+        if diff_ids.any?
+          MergeRequestDiff.transaction do
+            Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows)
+            Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows)
+
+            MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil)
+          end
+        end
+
+        reset_buffers!
+      end
+
+      def single_diff_rows(merge_request_diff)
+        sha_attribute = Gitlab::Database::ShaAttribute.new
+        commits = YAML.load(merge_request_diff.st_commits) rescue []
+
+        commit_rows = commits.map.with_index do |commit, index|
+          commit_hash = commit.to_hash.with_indifferent_access.except(:parent_ids)
+          sha = commit_hash.delete(:id)
+
+          commit_hash.merge(
+            merge_request_diff_id: merge_request_diff.id,
+            relative_order: index,
+            sha: sha_attribute.type_cast_for_database(sha)
+          )
+        end
+
+        diffs = YAML.load(merge_request_diff.st_diffs) rescue []
+        diffs = [] unless valid_raw_diffs?(diffs)
+
+        file_rows = diffs.map.with_index do |diff, index|
+          diff_hash = diff.to_hash.with_indifferent_access.merge(
+            binary: false,
+            merge_request_diff_id: merge_request_diff.id,
+            relative_order: index
+          )
+
+          # Compatibility with old diffs created with Psych.
+          diff_hash.tap do |hash|
+            diff_text = hash[:diff]
+
+            if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only?
+              hash[:binary] = true
+              hash[:diff] = [diff_text].pack('m0')
+            end
+          end
+        end
+
+        [commit_rows, file_rows]
+      end
+
+      # Unlike MergeRequestDiff#valid_raw_diff?, don't count Rugged objects as
+      # valid, because we don't render them usefully anyway.
+      def valid_raw_diffs?(diffs)
+        return false unless diffs.respond_to?(:each)
+
+        diffs.all? { |diff| diff.is_a?(Hash) }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb
index bf557103cfd..0735243e021 100644
--- a/lib/gitlab/contributions_calendar.rb
+++ b/lib/gitlab/contributions_calendar.rb
@@ -48,7 +48,7 @@ module Gitlab
     end
 
     def starting_month
-      Date.today.month
+      Date.current.month
     end
 
     private
@@ -66,12 +66,18 @@ module Gitlab
         .select(:id)
 
       conditions = t[:created_at].gteq(date_from.beginning_of_day)
-        .and(t[:created_at].lteq(Date.today.end_of_day))
+        .and(t[:created_at].lteq(Date.current.end_of_day))
         .and(t[:author_id].eq(contributor.id))
 
+      date_interval = if Gitlab::Database.postgresql?
+                        "INTERVAL '#{Time.zone.now.utc_offset} seconds'"
+                      else
+                        "INTERVAL #{Time.zone.now.utc_offset} SECOND"
+                      end
+
       Event.reorder(nil)
-        .select(t[:project_id], t[:target_type], t[:action], 'date(created_at) AS date', 'count(id) as total_amount')
-        .group(t[:project_id], t[:target_type], t[:action], 'date(created_at)')
+        .select(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval}) AS date", 'count(id) as total_amount')
+        .group(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval})")
         .where(conditions)
         .having(t[:project_id].in(Arel::Nodes::SqlLiteral.new(authed_projects.to_sql)))
     end
diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb
index 31579e94a87..8eea33b9ab5 100644
--- a/lib/gitlab/email/handler/create_note_handler.rb
+++ b/lib/gitlab/email/handler/create_note_handler.rb
@@ -15,7 +15,6 @@ module Gitlab
 
         def execute
           raise SentNotificationNotFoundError unless sent_notification
-          raise AutoGeneratedEmailError if mail.header.to_s =~ /auto-(generated|replied)/
 
           validate_permission!(:create_note)
 
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index 0d6b08b5d29..c8f4591d060 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -26,6 +26,9 @@ module Gitlab
         raise EmptyEmailError if @raw.blank?
 
         mail = build_mail
+
+        ignore_auto_submitted!(mail)
+
         mail_key = extract_mail_key(mail)
         handler = Handler.for(mail, mail_key)
 
@@ -87,6 +90,16 @@ module Gitlab
           break key if key
         end
       end
+
+      def ignore_auto_submitted!(mail)
+        # Mail::Header#[] is case-insensitive
+        auto_submitted = mail.header['Auto-Submitted']&.value
+
+        # Mail::Field#value would strip leading and trailing whitespace
+        raise AutoGeneratedEmailError if
+          # See also https://tools.ietf.org/html/rfc3834
+          auto_submitted && auto_submitted != 'no'
+      end
     end
   end
 end
diff --git a/lib/gitlab/environment.rb b/lib/gitlab/environment.rb
new file mode 100644
index 00000000000..5e0dd6e7859
--- /dev/null
+++ b/lib/gitlab/environment.rb
@@ -0,0 +1,7 @@
+module Gitlab
+  module Environment
+    def self.hostname
+      @hostname ||= ENV['HOSTNAME'] || Socket.gethostname
+    end
+  end
+end
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
index 0deaab01b5b..8dbe25e55f6 100644
--- a/lib/gitlab/git/blame.rb
+++ b/lib/gitlab/git/blame.rb
@@ -1,5 +1,3 @@
-# Gitaly note: JV: needs 1 RPC for #load_blame.
-
 module Gitlab
   module Git
     class Blame
@@ -26,15 +24,29 @@ module Gitlab
 
       private
 
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/376
       def load_blame
-        cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path})
-        # Read in binary mode to ensure ASCII-8BIT
-        raw_output = IO.popen(cmd, 'rb') {|io| io.read }
+        raw_output = @repo.gitaly_migrate(:blame) do |is_enabled|
+          if is_enabled
+            load_blame_by_gitaly
+          else
+            load_blame_by_shelling_out
+          end
+        end
+
         output = encode_utf8(raw_output)
         process_raw_blame output
       end
 
+      def load_blame_by_gitaly
+        @repo.gitaly_commit_client.raw_blame(@sha, @path)
+      end
+
+      def load_blame_by_shelling_out
+        cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path})
+        # Read in binary mode to ensure ASCII-8BIT
+        IO.popen(cmd, 'rb') {|io| io.read }
+      end
+
       def process_raw_blame(output)
         lines, final = [], []
         info, commits = {}, {}
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index ca7e3a7c4be..600d886e818 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -102,7 +102,7 @@ module Gitlab
             if is_enabled
               repo.gitaly_commit_client.between(base, head)
             else
-              repo.commits_between(base, head).map { |c| decorate(c) }
+              repo.rugged_commits_between(base, head).map { |c| decorate(c) }
             end
           end
         rescue Rugged::ReferenceError
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 70d793f8e4a..1005a819f95 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -58,17 +58,18 @@ module Gitlab
         end
       end
 
-      # Alias to old method for compatibility
-      def raw
-        rugged
-      end
-
       def rugged
-        @rugged ||= Rugged::Repository.new(path, alternates: alternate_object_directories)
+        @rugged ||= circuit_breaker.perform do
+          Rugged::Repository.new(path, alternates: alternate_object_directories)
+        end
       rescue Rugged::RepositoryError, Rugged::OSError
         raise NoRepository.new('no repository for such path')
       end
 
+      def circuit_breaker
+        @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage)
+      end
+
       # Returns an Array of branch names
       # sorted by name ASC
       def branch_names
@@ -296,21 +297,19 @@ module Gitlab
       #     after: Time.new(2016, 4, 21, 14, 32, 10)
       #   )
       #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/446
       def log(options)
         raw_log(options).map { |c| Commit.decorate(c) }
       end
 
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/382
       def count_commits(options)
-        cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list]
-        cmd << "--after=#{options[:after].iso8601}" if options[:after]
-        cmd << "--before=#{options[:before].iso8601}" if options[:before]
-        cmd += %W[--count #{options[:ref]}]
-        cmd += %W[-- #{options[:path]}] if options[:path].present?
-
-        raw_output = IO.popen(cmd) { |io| io.read }
-
-        raw_output.to_i
+        gitaly_migrate(:count_commits) do |is_enabled|
+          if is_enabled
+            count_commits_by_gitaly(options)
+          else
+            count_commits_by_shelling_out(options)
+          end
+        end
       end
 
       def sha_from_ref(ref)
@@ -327,7 +326,9 @@ module Gitlab
       # Return a collection of Rugged::Commits between the two revspec arguments.
       # See http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for
       # a detailed list of valid arguments.
-      def commits_between(from, to)
+      #
+      # Gitaly note: JV: to be deprecated in favor of Commit.between
+      def rugged_commits_between(from, to)
         walker = Rugged::Walker.new(rugged)
         walker.sorting(Rugged::SORT_NONE | Rugged::SORT_REVERSE)
 
@@ -686,6 +687,14 @@ module Gitlab
         @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self)
       end
 
+      def gitaly_migrate(method, &block)
+        Gitlab::GitalyClient.migrate(method, &block)
+      rescue GRPC::NotFound => e
+        raise NoRepository.new(e)
+      rescue GRPC::BadStatus => e
+        raise CommandError.new(e)
+      end
+
       private
 
       # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'.
@@ -852,46 +861,6 @@ module Gitlab
         submodule_data.select { |path, data| data['id'] }
       end
 
-      # Returns true if +commit+ introduced changes to +path+, using commit
-      # trees to make that determination.  Uses the history simplification
-      # rules that `git log` uses by default, where a commit is omitted if it
-      # is TREESAME to any parent.
-      #
-      # If the +follow+ option is true and the file specified by +path+ was
-      # renamed, then the path value is set to the old path.
-      def commit_touches_path?(commit, path, follow, walker)
-        entry = tree_entry(commit, path)
-
-        if commit.parents.empty?
-          # This is the root commit, return true if it has +path+ in its tree
-          return !entry.nil?
-        end
-
-        num_treesame = 0
-        commit.parents.each do |parent|
-          parent_entry = tree_entry(parent, path)
-
-          # Only follow the first TREESAME parent for merge commits
-          if num_treesame > 0
-            walker.hide(parent)
-            next
-          end
-
-          if entry.nil? && parent_entry.nil?
-            num_treesame += 1
-          elsif entry && parent_entry && entry[:oid] == parent_entry[:oid]
-            num_treesame += 1
-          end
-        end
-
-        case num_treesame
-        when 0
-          detect_rename(commit, commit.parents.first, path) if follow
-          true
-        else false
-        end
-      end
-
       # Find the entry for +path+ in the tree for +commit+
       def tree_entry(commit, path)
         pathname = Pathname.new(path)
@@ -919,43 +888,6 @@ module Gitlab
         tmp_entry
       end
 
-      # Compare +commit+ and +parent+ for +path+.  If +path+ is a file and was
-      # renamed in +commit+, then set +path+ to the old filename.
-      def detect_rename(commit, parent, path)
-        diff = parent.diff(commit, paths: [path], disable_pathspec_match: true)
-
-        # If +path+ is a filename, not a directory, then we should only have
-        # one delta.  We don't need to follow renames for directories.
-        return nil if diff.each_delta.count > 1
-
-        delta = diff.each_delta.first
-        if delta.added?
-          full_diff = parent.diff(commit)
-          full_diff.find_similar!
-
-          full_diff.each_delta do |full_delta|
-            if full_delta.renamed? && path == full_delta.new_file[:path]
-              # Look for the old path in ancestors
-              path.replace(full_delta.old_file[:path])
-            end
-          end
-        end
-      end
-
-      # Returns true if the index entry has the special file mode that denotes
-      # a submodule.
-      def submodule?(index_entry)
-        index_entry[:mode] == 57344
-      end
-
-      # Return a Rugged::Index that has read from the tree at +ref_name+
-      def populated_index(ref_name)
-        commit = rev_parse_target(ref_name)
-        index = rugged.index
-        index.read_tree(commit.tree)
-        index
-      end
-
       # Return the Rugged patches for the diff between +from+ and +to+.
       def diff_patches(from, to, options = {}, *paths)
         options ||= {}
@@ -1001,16 +933,29 @@ module Gitlab
         end.sort_by(&:name)
       end
 
+      def last_commit_for_path_by_rugged(sha, path)
+        sha = last_commit_id_for_path(sha, path)
+        commit(sha)
+      end
+
       def tags_from_gitaly
         gitaly_ref_client.tags
       end
 
-      def gitaly_migrate(method, &block)
-        Gitlab::GitalyClient.migrate(method, &block)
-      rescue GRPC::NotFound => e
-        raise NoRepository.new(e)
-      rescue GRPC::BadStatus => e
-        raise CommandError.new(e)
+      def count_commits_by_gitaly(options)
+        gitaly_commit_client.commit_count(options[:ref], options)
+      end
+
+      def count_commits_by_shelling_out(options)
+        cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list]
+        cmd << "--after=#{options[:after].iso8601}" if options[:after]
+        cmd << "--before=#{options[:before].iso8601}" if options[:before]
+        cmd += %W[--count #{options[:ref]}]
+        cmd += %W[-- #{options[:path]}] if options[:path].present?
+
+        raw_output = IO.popen(cmd) { |io| io.read }
+
+        raw_output.to_i
       end
     end
   end
diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb
new file mode 100644
index 00000000000..e28be4b8a38
--- /dev/null
+++ b/lib/gitlab/git/storage.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Git
+    module Storage
+      class Inaccessible < StandardError
+        attr_reader :retry_after
+
+        def initialize(message = nil, retry_after = nil)
+          super(message)
+          @retry_after = retry_after
+        end
+      end
+
+      CircuitOpen = Class.new(Inaccessible)
+
+      REDIS_KEY_PREFIX = 'storage_accessible:'.freeze
+
+      def self.redis
+        Gitlab::Redis::SharedState
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb
new file mode 100644
index 00000000000..9ea9367d4b7
--- /dev/null
+++ b/lib/gitlab/git/storage/circuit_breaker.rb
@@ -0,0 +1,144 @@
+module Gitlab
+  module Git
+    module Storage
+      class CircuitBreaker
+        FailureInfo = Struct.new(:last_failure, :failure_count)
+
+        attr_reader :storage,
+                    :hostname,
+                    :storage_path,
+                    :failure_count_threshold,
+                    :failure_wait_time,
+                    :failure_reset_time,
+                    :storage_timeout
+
+        delegate :last_failure, :failure_count, to: :failure_info
+
+        def self.reset_all!
+          pattern = "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}*"
+
+          Gitlab::Git::Storage.redis.with do |redis|
+            all_storage_keys = redis.keys(pattern)
+            redis.del(*all_storage_keys) unless all_storage_keys.empty?
+          end
+
+          RequestStore.delete(:circuitbreaker_cache)
+        end
+
+        def self.for_storage(storage)
+          cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do
+            Hash.new do |hash, storage_name|
+              hash[storage_name] = new(storage_name)
+            end
+          end
+
+          cached_circuitbreakers[storage]
+        end
+
+        def initialize(storage, hostname = Gitlab::Environment.hostname)
+          @storage = storage
+          @hostname = hostname
+
+          config = Gitlab.config.repositories.storages[@storage]
+          @storage_path = config['path']
+          @failure_count_threshold = config['failure_count_threshold']
+          @failure_wait_time = config['failure_wait_time']
+          @failure_reset_time = config['failure_reset_time']
+          @storage_timeout = config['storage_timeout']
+        end
+
+        def perform
+          return yield unless Feature.enabled?('git_storage_circuit_breaker')
+
+          check_storage_accessible!
+
+          yield
+        end
+
+        def circuit_broken?
+          return false if no_failures?
+
+          recent_failure = last_failure > failure_wait_time.seconds.ago
+          too_many_failures = failure_count > failure_count_threshold
+
+          recent_failure || too_many_failures
+        end
+
+        # Memoizing the `storage_available` call means we only do it once per
+        # request when the storage is available.
+        #
+        # When the storage appears not available, and the memoized value is `false`
+        # we might want to try again.
+        def storage_available?
+          return @storage_available if @storage_available
+
+          if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck
+                                    .storage_available?(storage_path, storage_timeout)
+            track_storage_accessible
+          else
+            track_storage_inaccessible
+          end
+
+          @storage_available
+        end
+
+        def check_storage_accessible!
+          if circuit_broken?
+            raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_wait_time)
+          end
+
+          unless storage_available?
+            raise Gitlab::Git::Storage::Inaccessible.new("#{storage} not accessible", failure_wait_time)
+          end
+        end
+
+        def no_failures?
+          last_failure.blank? && failure_count == 0
+        end
+
+        def track_storage_inaccessible
+          @failure_info = FailureInfo.new(Time.now, failure_count + 1)
+
+          Gitlab::Git::Storage.redis.with do |redis|
+            redis.pipelined do
+              redis.hset(cache_key, :last_failure, last_failure.to_i)
+              redis.hincrby(cache_key, :failure_count, 1)
+              redis.expire(cache_key, failure_reset_time)
+            end
+          end
+        end
+
+        def track_storage_accessible
+          return if no_failures?
+
+          @failure_info = FailureInfo.new(nil, 0)
+
+          Gitlab::Git::Storage.redis.with do |redis|
+            redis.pipelined do
+              redis.hset(cache_key, :last_failure, nil)
+              redis.hset(cache_key, :failure_count, 0)
+            end
+          end
+        end
+
+        def failure_info
+          @failure_info ||= get_failure_info
+        end
+
+        def get_failure_info
+          last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis|
+            redis.hmget(cache_key, :last_failure, :failure_count)
+          end
+
+          last_failure = Time.at(last_failure.to_i) if last_failure.present?
+
+          FailureInfo.new(last_failure, failure_count.to_i)
+        end
+
+        def cache_key
+          @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb
new file mode 100644
index 00000000000..91d8241f17b
--- /dev/null
+++ b/lib/gitlab/git/storage/forked_storage_check.rb
@@ -0,0 +1,55 @@
+module Gitlab
+  module Git
+    module Storage
+      module ForkedStorageCheck
+        extend self
+
+        def storage_available?(path, timeout_seconds = 5)
+          status = timeout_check(path, timeout_seconds)
+
+          status.success?
+        end
+
+        def timeout_check(path, timeout_seconds)
+          filesystem_check_pid = check_filesystem_in_process(path)
+
+          deadline = timeout_seconds.seconds.from_now.utc
+          wait_time = 0.01
+          status = nil
+
+          while status.nil?
+            if deadline > Time.now.utc
+              sleep(wait_time)
+              _pid, status = Process.wait2(filesystem_check_pid, Process::WNOHANG)
+            else
+              Process.kill('KILL', filesystem_check_pid)
+              # Blocking wait, so we are sure the process is gone before continuing
+              _pid, status = Process.wait2(filesystem_check_pid)
+            end
+          end
+
+          status
+        end
+
+        # This will spawn a new 2 processes to do the check:
+        # The outer child (waiter) will spawn another child process (stater).
+        #
+        # The stater is the process is performing the actual filesystem check
+        # the check might hang if the filesystem is acting up.
+        # In this case we will send a `KILL` to the waiter, which will still
+        # be responsive while the stater is hanging.
+        def check_filesystem_in_process(path)
+          spawn('ruby', '-e', ruby_check, path, [:out, :err] => '/dev/null')
+        end
+
+        def ruby_check
+          <<~RUBY_FILESYSTEM_CHECK
+          inner_pid = fork { File.stat(ARGV.first) }
+          Process.waitpid(inner_pid)
+          exit $?.exitstatus
+          RUBY_FILESYSTEM_CHECK
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb
new file mode 100644
index 00000000000..2d723147f4f
--- /dev/null
+++ b/lib/gitlab/git/storage/health.rb
@@ -0,0 +1,91 @@
+module Gitlab
+  module Git
+    module Storage
+      class Health
+        attr_reader :storage_name, :info
+
+        def self.pattern_for_storage(storage_name)
+          "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage_name}:*"
+        end
+
+        def self.for_all_storages
+          storage_names = Gitlab.config.repositories.storages.keys
+          results_per_storage = nil
+
+          Gitlab::Git::Storage.redis.with do |redis|
+            keys_per_storage = all_keys_for_storages(storage_names, redis)
+            results_per_storage = load_for_keys(keys_per_storage, redis)
+          end
+
+          results_per_storage.map do |name, info|
+            info.each { |i| i[:failure_count] = i[:failure_count].value.to_i }
+            new(name, info)
+          end
+        end
+
+        def self.all_keys_for_storages(storage_names, redis)
+          keys_per_storage = {}
+
+          redis.pipelined do
+            storage_names.each do |storage_name|
+              pattern = pattern_for_storage(storage_name)
+
+              keys_per_storage[storage_name] = redis.keys(pattern)
+            end
+          end
+
+          keys_per_storage
+        end
+
+        def self.load_for_keys(keys_per_storage, redis)
+          info_for_keys = {}
+
+          redis.pipelined do
+            keys_per_storage.each do |storage_name, keys_future|
+              info_for_storage = keys_future.value.map do |key|
+                { name: key, failure_count: redis.hget(key, :failure_count) }
+              end
+
+              info_for_keys[storage_name] = info_for_storage
+            end
+          end
+
+          info_for_keys
+        end
+
+        def self.for_failing_storages
+          for_all_storages.select(&:failing?)
+        end
+
+        def initialize(storage_name, info)
+          @storage_name = storage_name
+          @info = info
+        end
+
+        def failing_info
+          @failing_info ||= info.select { |info_for_host| info_for_host[:failure_count] > 0 }
+        end
+
+        def failing?
+          failing_info.any?
+        end
+
+        def failing_on_hosts
+          @failing_on_hosts ||= failing_info.map do |info_for_host|
+            info_for_host[:name].split(':').last
+          end
+        end
+
+        def failing_circuit_breakers
+          @failing_circuit_breakers ||= failing_on_hosts.map do |hostname|
+            CircuitBreaker.new(storage_name, hostname)
+          end
+        end
+
+        def total_failures
+          @total_failures ||= failing_info.sum { |info_for_host| info_for_host[:failure_count] }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index a834781b1f1..ac6817e6d0e 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -85,15 +85,32 @@ module Gitlab
         end
       end
 
-      def commit_count(ref)
+      def commit_count(ref, options = {})
         request = Gitaly::CountCommitsRequest.new(
           repository: @gitaly_repo,
           revision: ref
         )
+        request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present?
+        request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present?
+        request.path = options[:path] if options[:path].present?
 
         GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count
       end
 
+      def last_commit_for_path(revision, path)
+        request = Gitaly::LastCommitForPathRequest.new(
+          repository: @gitaly_repo,
+          revision: revision.force_encoding(Encoding::ASCII_8BIT),
+          path: path.to_s.force_encoding(Encoding::ASCII_8BIT)
+        )
+
+        gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request).commit
+        return unless gitaly_commit
+
+        commit = GitalyClient::Commit.new(@repository, gitaly_commit)
+        Gitlab::Git::Commit.new(commit)
+      end
+
       def between(from, to)
         request = Gitaly::CommitsBetweenRequest.new(
           repository: @gitaly_repo,
@@ -125,6 +142,17 @@ module Gitlab
         response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } }
       end
 
+      def raw_blame(revision, path)
+        request = Gitaly::RawBlameRequest.new(
+          repository: @gitaly_repo,
+          revision: revision,
+          path: path
+        )
+
+        response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request)
+        response.reduce("") { |memo, msg| memo << msg.data }
+      end
+
       private
 
       def commit_diff_request_params(commit, options = {})
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb
index 9e91c135956..eef97f54962 100644
--- a/lib/gitlab/health_checks/fs_shards_check.rb
+++ b/lib/gitlab/health_checks/fs_shards_check.rb
@@ -10,7 +10,9 @@ module Gitlab
         def readiness
           repository_storages.map do |storage_name|
             begin
-              if !storage_stat_test(storage_name)
+              if !storage_circuitbreaker_test(storage_name)
+                HealthChecks::Result.new(false, 'circuitbreaker tripped', shard: storage_name)
+              elsif !storage_stat_test(storage_name)
                 HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name)
               else
                 with_temp_file(storage_name) do |tmp_file_path|
@@ -36,7 +38,8 @@ module Gitlab
             [
               storage_stat_metrics(storage_name),
               storage_write_metrics(storage_name),
-              storage_read_metrics(storage_name)
+              storage_read_metrics(storage_name),
+              storage_circuitbreaker_metrics(storage_name)
             ].flatten
           end
         end
@@ -121,6 +124,12 @@ module Gitlab
           file_contents == RANDOM_STRING
         end
 
+        def storage_circuitbreaker_test(storage_name)
+          Gitlab::Git::Storage::CircuitBreaker.new(storage_name).perform { "OK" }
+        rescue Gitlab::Git::Storage::Inaccessible
+          nil
+        end
+
         def storage_stat_metrics(storage_name)
           operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, shard: storage_name) do
             with_timing { storage_stat_test(storage_name) }
@@ -143,6 +152,14 @@ module Gitlab
             end
           end
         end
+
+        def storage_circuitbreaker_metrics(storage_name)
+          operation_metrics(:filesystem_circuitbreaker,
+                            :filesystem_circuitbreaker_latency_seconds,
+                            shard: storage_name) do
+            with_timing { storage_circuitbreaker_test(storage_name) }
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
index cc282d1415b..5d106b5c075 100644
--- a/lib/gitlab/i18n.rb
+++ b/lib/gitlab/i18n.rb
@@ -16,7 +16,8 @@ module Gitlab
       'eo' => 'Esperanto',
       'it' => 'Italiano',
       'uk' => 'Українська',
-      'ja' => '日本語'
+      'ja' => '日本語',
+      'ko' => '한국어'
     }.freeze
 
     def available_locales
diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
index db4708b22e4..32fe8201a8d 100644
--- a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
+++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
@@ -5,7 +5,7 @@ module Gitlab
         include QueryAdditionalMetrics
 
         def query(environment_id)
-          Environment.find_by(id: environment_id).try do |environment|
+          ::Environment.find_by(id: environment_id).try do |environment|
             query_metrics(
               common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f)
             )
diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb
index 66f29d95177..1d17d3cfd56 100644
--- a/lib/gitlab/prometheus/queries/environment_query.rb
+++ b/lib/gitlab/prometheus/queries/environment_query.rb
@@ -3,7 +3,7 @@ module Gitlab
     module Queries
       class EnvironmentQuery < BaseQuery
         def query(environment_id)
-          Environment.find_by(id: environment_id).try do |environment|
+          ::Environment.find_by(id: environment_id).try do |environment|
             environment_slug = environment.slug
             timeframe_start = 8.hours.ago.to_f
             timeframe_end = Time.now.to_f
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index e0ac21305a5..748e0a29184 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -27,8 +27,8 @@ module Gitlab
             ci_pipeline_schedules: ::Ci::PipelineSchedule.count,
             deploy_keys: DeployKey.count,
             deployments: Deployment.count,
-            environments: Environment.count,
-            in_review_folder: Environment.in_review_folder.count,
+            environments: ::Environment.count,
+            in_review_folder: ::Environment.in_review_folder.count,
             groups: Group.count,
             issues: Issue.count,
             keys: Key.count,
diff --git a/lib/haml_lint/inline_javascript.rb b/lib/haml_lint/inline_javascript.rb
new file mode 100644
index 00000000000..f3ddcbb9c95
--- /dev/null
+++ b/lib/haml_lint/inline_javascript.rb
@@ -0,0 +1,14 @@
+require 'haml_lint/haml_visitor'
+require 'haml_lint/linter'
+require 'haml_lint/linter_registry'
+
+module HamlLint
+  class Linter::InlineJavaScript < Linter
+    include LinterRegistry
+
+    def visit_filter(node)
+      return unless node.filter_type == 'javascript'
+      record_lint(node, 'Inline JavaScript is discouraged (https://docs.gitlab.com/ee/development/gotchas.html#do-not-use-inline-javascript-in-views)')
+    end
+  end
+end
diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb
index 688a79c0441..ef08bd46e17 100644
--- a/lib/mattermost/session.rb
+++ b/lib/mattermost/session.rb
@@ -36,11 +36,12 @@ module Mattermost
 
     def with_session
       with_lease do
-        raise Mattermost::NoSessionError unless create
+        create
 
         begin
           yield self
-        rescue Errno::ECONNREFUSED
+        rescue Errno::ECONNREFUSED => e
+          Rails.logger.error(e.message + "\n" + e.backtrace.join("\n"))
           raise Mattermost::NoSessionError
         ensure
           destroy
@@ -85,10 +86,12 @@ module Mattermost
     private
 
     def create
-      return unless oauth_uri
-      return unless token_uri
+      raise Mattermost::NoSessionError unless oauth_uri
+      raise Mattermost::NoSessionError unless token_uri
 
       @token = request_token
+      raise Mattermost::NoSessionError unless @token
+
       @headers = {
         Authorization: "Bearer #{@token}"
       }
@@ -106,11 +109,16 @@ module Mattermost
       @oauth_uri = nil
 
       response = get("/api/v3/oauth/gitlab/login", follow_redirects: false)
-      return unless 300 <= response.code && response.code < 400
+      return unless (300...400) === response.code
 
       redirect_uri = response.headers['location']
       return unless redirect_uri
 
+      oauth_cookie = parse_cookie(response)
+      @headers = {
+        Cookie: oauth_cookie.to_cookie_string
+      }
+
       @oauth_uri = URI.parse(redirect_uri)
     end
 
@@ -124,7 +132,7 @@ module Mattermost
     def request_token
       response = get(token_uri, follow_redirects: false)
 
-      if 200 <= response.code && response.code < 400
+      if (200...400) === response.code
         response.headers['token']
       end
     end
@@ -156,5 +164,11 @@ module Mattermost
     rescue Errno::ECONNREFUSED => e
       raise Mattermost::ConnectionError.new(e.message)
     end
+
+    def parse_cookie(response)
+      cookie_hash = CookieHash.new
+      response.get_fields('Set-Cookie').each { |c| cookie_hash.add_cookies(c) }
+      cookie_hash
+    end
   end
 end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index 680e76af471..7d35e0df53a 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -21,7 +21,7 @@ namespace :gitlab do
         create_gitaly_configuration
         # In CI we run scripts/gitaly-test-build instead of this command
         unless ENV['CI'].present?
-          Bundler.with_original_env { run_command!(%w[/usr/bin/env -u BUNDLE_GEMFILE] + [command]) }
+          Bundler.with_original_env { run_command!([command]) }
         end
       end
     end
@@ -66,6 +66,7 @@ namespace :gitlab do
       config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages }
       config[:auth] = { token: 'secret' } if Rails.env.test?
       config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby
+      config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
       TOML.dump(config)
     end
 
diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake
index 609dfaa48e3..ad2d034b0b4 100644
--- a/lib/tasks/haml-lint.rake
+++ b/lib/tasks/haml-lint.rake
@@ -1,5 +1,6 @@
 unless Rails.env.production?
   require 'haml_lint/rake_task'
+  require 'haml_lint/inline_javascript'
 
   HamlLint::RakeTask.new
 end