Merge branch 'master' into sh-headless-chrome-support

* master: (96 commits)
  Fetch the merged branches at once
  Merging EE doc into CE
  Avoid using Rugged in Gitlab::Git::Wiki#preview_slug
  Cache commits on the repository model
  Remove groups_select from global namespace & simplifies the code
  Change default disabled merge request widget message to "Merge is not allowed yet"
  Semi-linear history merge is now available in CE.
  Remove repetitive karma spec
  Improve spec to check hidden component
  Rename to shouldShowUsername
  Add KubernetesService#default_namespace tests
  Revert "Merge branch '36670-remove-edit-form' into 'master'"
  Fix bitbucket login
  Remove duped tests
  Add path attribute to WikiFile class
  Make local_branches OPT_OUT
  Clarify the language around External Group membership with SAML SSO to clarify that this will NOT add users to GitLab Groups.
  Added ssh fingerprint, gitlab ci and pages information in an instance configuration page
  Fix the incorrect value being used to set GL_USERNAME on hooks
  Resolve "Remove overzealous tooltips in projects page tabs"
  ...

18 files changed, 130 insertions, 43 deletions

diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb
index c53882787f1..3487e099381 100644
--- a/lib/gitlab/git/branch.rb
+++ b/lib/gitlab/git/branch.rb
@@ -3,6 +3,14 @@
 module Gitlab
   module Git
     class Branch < Ref
+      def self.find(repo, branch_name)
+        if branch_name.is_a?(Gitlab::Git::Branch)
+          branch_name
+        else
+          repo.find_branch(branch_name)
+        end
+      end
+
       def initialize(repository, name, target, target_commit)
         super(repository, name, target, target_commit)
       end
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 1957c254c28..23ae37ff71e 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -72,7 +72,8 @@ module Gitlab
 
           decorate(repo, commit) if commit
         rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError,
-               Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository
+               Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository,
+               Rugged::OdbError, Rugged::TreeError
           nil
         end
 
diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb
index c327e9b1616..f302b852b35 100644
--- a/lib/gitlab/git/hooks_service.rb
+++ b/lib/gitlab/git/hooks_service.rb
@@ -8,7 +8,7 @@ module Gitlab
       def execute(pusher, repository, oldrev, newrev, ref)
         @repository  = repository
         @gl_id       = pusher.gl_id
-        @gl_username = pusher.name
+        @gl_username = pusher.username
         @oldrev      = oldrev
         @newrev      = newrev
         @ref         = ref
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 59a54b48ed9..fc8af38d4d9 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -166,7 +166,7 @@ module Gitlab
       end
 
       def local_branches(sort_by: nil)
-        gitaly_migrate(:local_branches) do |is_enabled|
+        gitaly_migrate(:local_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
           if is_enabled
             gitaly_ref_client.local_branches(sort_by: sort_by)
           else
@@ -511,6 +511,10 @@ module Gitlab
         gitaly_commit_client.ancestor?(from, to)
       end
 
+      def merged_branch_names(branch_names = [])
+        Set.new(git_merged_branch_names(branch_names))
+      end
+
       # Return an array of Diff objects that represent the diff
       # between +from+ and +to+.  See Diff::filter_diff_options for the allowed
       # diff options.  The +options+ hash can also include :break_rewrites to
@@ -745,6 +749,16 @@ module Gitlab
         nil
       end
 
+      def ff_merge(user, source_sha, target_branch)
+        OperationService.new(user, self).with_branch(target_branch) do |our_commit|
+          raise ArgumentError, 'Invalid merge target' unless our_commit
+
+          source_sha
+        end
+      rescue Rugged::ReferenceError
+        raise ArgumentError, 'Invalid merge source'
+      end
+
       def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
         OperationService.new(user, self).with_branch(
           branch_name,
@@ -1180,6 +1194,13 @@ module Gitlab
         sort_branches(branches, sort_by)
       end
 
+      def git_merged_branch_names(branch_names = [])
+        lines = run_git(['branch', '--merged', root_ref] + branch_names)
+          .first.lines
+
+        lines.map(&:strip)
+      end
+
       def log_using_shell?(options)
         options[:path].present? ||
           options[:disable_walk] ||
diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb
index 08e6c29abad..99518c9b1e4 100644
--- a/lib/gitlab/git/storage.rb
+++ b/lib/gitlab/git/storage.rb
@@ -12,6 +12,7 @@ module Gitlab
 
       CircuitOpen = Class.new(Inaccessible)
       Misconfiguration = Class.new(Inaccessible)
+      Failing = Class.new(Inaccessible)
 
       REDIS_KEY_PREFIX = 'storage_accessible:'.freeze
 
diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb
index 0456ad9a1f3..be7598ef011 100644
--- a/lib/gitlab/git/storage/circuit_breaker.rb
+++ b/lib/gitlab/git/storage/circuit_breaker.rb
@@ -54,7 +54,7 @@ module Gitlab
         end
 
         def perform
-          return yield unless Feature.enabled?('git_storage_circuit_breaker')
+          return yield unless enabled?
 
           check_storage_accessible!
 
@@ -64,10 +64,27 @@ module Gitlab
         def circuit_broken?
           return false if no_failures?
 
+          failure_count > failure_count_threshold
+        end
+
+        def backing_off?
+          return false if no_failures?
+
           recent_failure = last_failure > failure_wait_time.seconds.ago
-          too_many_failures = failure_count > failure_count_threshold
+          too_many_failures = failure_count > backoff_threshold
 
-          recent_failure || too_many_failures
+          recent_failure && too_many_failures
+        end
+
+        private
+
+        # The circuitbreaker can be enabled for the entire fleet using a Feature
+        # flag.
+        #
+        # Enabling it for a single host can be done setting the
+        # `GIT_STORAGE_CIRCUIT_BREAKER` environment variable.
+        def enabled?
+          ENV['GIT_STORAGE_CIRCUIT_BREAKER'].present? || Feature.enabled?('git_storage_circuit_breaker')
         end
 
         def failure_info
@@ -83,7 +100,7 @@ module Gitlab
           return @storage_available if @storage_available
 
           if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck
-                                    .storage_available?(storage_path, storage_timeout)
+                                    .storage_available?(storage_path, storage_timeout, access_retries)
             track_storage_accessible
           else
             track_storage_inaccessible
@@ -94,7 +111,11 @@ module Gitlab
 
         def check_storage_accessible!
           if circuit_broken?
-            raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_wait_time)
+            raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time)
+          end
+
+          if backing_off?
+            raise Gitlab::Git::Storage::Failing.new("Backing off access to #{storage}", failure_wait_time)
           end
 
           unless storage_available?
@@ -131,12 +152,6 @@ module Gitlab
           end
         end
 
-        def cache_key
-          @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
-        end
-
-        private
-
         def get_failure_info
           last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis|
             redis.hmget(cache_key, :last_failure, :failure_count)
@@ -146,6 +161,10 @@ module Gitlab
 
           FailureInfo.new(last_failure, failure_count.to_i)
         end
+
+        def cache_key
+          @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
+        end
       end
     end
   end
diff --git a/lib/gitlab/git/storage/circuit_breaker_settings.rb b/lib/gitlab/git/storage/circuit_breaker_settings.rb
index d2313fe7c1b..257fe8cd8f0 100644
--- a/lib/gitlab/git/storage/circuit_breaker_settings.rb
+++ b/lib/gitlab/git/storage/circuit_breaker_settings.rb
@@ -18,6 +18,14 @@ module Gitlab
           application_settings.circuitbreaker_storage_timeout
         end
 
+        def access_retries
+          application_settings.circuitbreaker_access_retries
+        end
+
+        def backoff_threshold
+          application_settings.circuitbreaker_backoff_threshold
+        end
+
         private
 
         def application_settings
diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb
index 91d8241f17b..1307f400700 100644
--- a/lib/gitlab/git/storage/forked_storage_check.rb
+++ b/lib/gitlab/git/storage/forked_storage_check.rb
@@ -4,8 +4,17 @@ module Gitlab
       module ForkedStorageCheck
         extend self
 
-        def storage_available?(path, timeout_seconds = 5)
-          status = timeout_check(path, timeout_seconds)
+        def storage_available?(path, timeout_seconds = 5, retries = 1)
+          partial_timeout = timeout_seconds / retries
+          status = timeout_check(path, partial_timeout)
+
+          # If the status check did not succeed the first time, we retry a few
+          # more times to avoid one-off failures
+          current_attempts = 1
+          while current_attempts < retries && !status.success?
+            status = timeout_check(path, partial_timeout)
+            current_attempts += 1
+          end
 
           status.success?
         end
diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb
index 60c6791a7e4..a12d52d295f 100644
--- a/lib/gitlab/git/storage/null_circuit_breaker.rb
+++ b/lib/gitlab/git/storage/null_circuit_breaker.rb
@@ -25,6 +25,10 @@ module Gitlab
           !!@error
         end
 
+        def backing_off?
+          false
+        end
+
         def last_failure
           circuit_broken? ? Time.now : nil
         end
diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb
index da74719ae87..e6b61417de1 100644
--- a/lib/gitlab/git/user.rb
+++ b/lib/gitlab/git/user.rb
@@ -7,9 +7,8 @@ module Gitlab
         new(gitlab_user.username, gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user))
       end
 
-      # TODO support the username field in Gitaly https://gitlab.com/gitlab-org/gitaly/issues/628
       def self.from_gitaly(gitaly_user)
-        new('', gitaly_user.name, gitaly_user.email, gitaly_user.gl_id)
+        new(gitaly_user.gl_username, gitaly_user.name, gitaly_user.email, gitaly_user.gl_id)
       end
 
       def initialize(username, name, email, gl_id)
@@ -22,6 +21,10 @@ module Gitlab
       def ==(other)
         [username, name, email, gl_id] == [other.username, other.name, other.email, other.gl_id]
       end
+
+      def to_gitaly
+        Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name, email: email)
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb
index e7b2f52a552..f01d5c96fc8 100644
--- a/lib/gitlab/git/wiki.rb
+++ b/lib/gitlab/git/wiki.rb
@@ -8,6 +8,7 @@ module Gitlab
           { name: name, email: email, message: message }
         end
       end
+      PageBlob = Struct.new(:name)
 
       def self.default_ref
         'master'
@@ -80,7 +81,15 @@ module Gitlab
       end
 
       def preview_slug(title, format)
-        gollum_wiki.preview_page(title, '', format).url_path
+        # Adapted from gollum gem (Gollum::Wiki#preview_page) to avoid
+        # using Rugged through a Gollum::Wiki instance
+        page_class = Gollum::Page
+        page = page_class.new(nil)
+        ext = page_class.format_to_ext(format.to_sym)
+        name = page_class.cname(title) + '.' + ext
+        blob = PageBlob.new(name)
+        page.populate(blob)
+        page.url_path
       end
 
       private
diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb
index 527f2a44dea..84335aca4bc 100644
--- a/lib/gitlab/git/wiki_file.rb
+++ b/lib/gitlab/git/wiki_file.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Git
     class WikiFile
-      attr_reader :mime_type, :raw_data, :name
+      attr_reader :mime_type, :raw_data, :name, :path
 
       # This class is meant to be serializable so that it can be constructed
       # by Gitaly and sent over the network to GitLab.
@@ -13,6 +13,7 @@ module Gitlab
         @mime_type = gollum_file.mime_type
         @raw_data = gollum_file.raw_data
         @name = gollum_file.name
+        @path = gollum_file.path
       end
     end
   end
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 42b59c106e2..8998c4b1a83 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -215,10 +215,6 @@ module Gitlab
       ).exec
     end
 
-    def matching_merge_request?(newrev, branch_name)
-      Checks::MatchingMergeRequest.new(newrev, branch_name, project).match?
-    end
-
     def deploy_key
       actor if deploy_key?
     end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 6c1ae19ff11..6868be26758 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -33,6 +33,12 @@ module Gitlab
     MUTEX = Mutex.new
     private_constant :MUTEX
 
+    class << self
+      attr_accessor :query_time
+    end
+
+    self.query_time = 0
+
     def self.stub(name, storage)
       MUTEX.synchronize do
         @stubs ||= {}
@@ -83,11 +89,14 @@ module Gitlab
     # end
     #
     def self.call(storage, service, rpc, request)
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       enforce_gitaly_request_limits(:call)
 
       kwargs = request_kwargs(storage)
       kwargs = yield(kwargs) if block_given?
       stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
+    ensure
+      self.query_time += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
     end
 
     def self.request_kwargs(storage)
diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb
index 91f34011f6e..adaf255f24b 100644
--- a/lib/gitlab/gitaly_client/operation_service.rb
+++ b/lib/gitlab/gitaly_client/operation_service.rb
@@ -10,7 +10,7 @@ module Gitlab
         request = Gitaly::UserDeleteTagRequest.new(
           repository: @gitaly_repo,
           tag_name: GitalyClient.encode(tag_name),
-          user: Util.gitaly_user(user)
+          user: Gitlab::Git::User.from_gitlab(user).to_gitaly
         )
 
         response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request)
@@ -23,7 +23,7 @@ module Gitlab
       def add_tag(tag_name, user, target, message)
         request = Gitaly::UserCreateTagRequest.new(
           repository: @gitaly_repo,
-          user: Util.gitaly_user(user),
+          user: Gitlab::Git::User.from_gitlab(user).to_gitaly,
           tag_name: GitalyClient.encode(tag_name),
           target_revision: GitalyClient.encode(target),
           message: GitalyClient.encode(message.to_s)
@@ -45,7 +45,7 @@ module Gitlab
         request = Gitaly::UserCreateBranchRequest.new(
           repository: @gitaly_repo,
           branch_name: GitalyClient.encode(branch_name),
-          user: Util.gitaly_user(user),
+          user: Gitlab::Git::User.from_gitlab(user).to_gitaly,
           start_point: GitalyClient.encode(start_point)
         )
         response = GitalyClient.call(@repository.storage, :operation_service,
@@ -65,7 +65,7 @@ module Gitlab
         request = Gitaly::UserDeleteBranchRequest.new(
           repository: @gitaly_repo,
           branch_name: GitalyClient.encode(branch_name),
-          user: Util.gitaly_user(user)
+          user: Gitlab::Git::User.from_gitlab(user).to_gitaly
         )
 
         response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request)
@@ -87,7 +87,7 @@ module Gitlab
         request_enum.push(
           Gitaly::UserMergeBranchRequest.new(
             repository: @gitaly_repo,
-            user: Util.gitaly_user(user),
+            user: Gitlab::Git::User.from_gitlab(user).to_gitaly,
             commit_id: source_sha,
             branch: GitalyClient.encode(target_branch),
             message: GitalyClient.encode(message)
diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb
index a1222a7e718..b1a033280b4 100644
--- a/lib/gitlab/gitaly_client/util.rb
+++ b/lib/gitlab/gitaly_client/util.rb
@@ -18,16 +18,6 @@ module Gitlab
           )
         end
 
-        def gitaly_user(gitlab_user)
-          return unless gitlab_user
-
-          Gitaly::User.new(
-            gl_id: Gitlab::GlId.gl_id(gitlab_user),
-            name: GitalyClient.encode(gitlab_user.name),
-            email: GitalyClient.encode(gitlab_user.email)
-          )
-        end
-
         def gitlab_tag_from_gitaly_tag(repository, gitaly_tag)
           if gitaly_tag.target_commit.present?
             commit = Gitlab::Git::Commit.decorate(repository, gitaly_tag.target_commit)
diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb
index 6bffd410ed0..a42e312b5d3 100644
--- a/lib/gitlab/logger.rb
+++ b/lib/gitlab/logger.rb
@@ -13,7 +13,7 @@ module Gitlab
     end
 
     def self.read_latest
-      path = Rails.root.join("log", file_name)
+      path = self.full_log_path
 
       return [] unless File.readable?(path)
 
@@ -22,7 +22,15 @@ module Gitlab
     end
 
     def self.build
-      new(Rails.root.join("log", file_name))
+      RequestStore[self.cache_key] ||= new(self.full_log_path)
+    end
+
+    def self.full_log_path
+      Rails.root.join("log", file_name)
+    end
+
+    def self.cache_key
+      'logger:'.freeze + self.full_log_path.to_s
     end
   end
 end
diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb
index 67fee8c227d..69e117f1da9 100644
--- a/lib/gitlab/performance_bar/peek_query_tracker.rb
+++ b/lib/gitlab/performance_bar/peek_query_tracker.rb
@@ -36,8 +36,8 @@ module Gitlab
       end
 
       def track_query(raw_query, bindings, start, finish)
-        query = Gitlab::Sherlock::Query.new(raw_query, start, finish)
-        query_info = { duration: query.duration.round(3), sql: query.formatted_query }
+        duration = finish - start
+        query_info = { duration: duration.round(3), sql: raw_query }
 
         PEEK_DB_CLIENT.query_details << query_info
       end