Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (170 commits)
  support ordering of project notes in notes api
  Redirect to an already forked project if it exists
  Reschedule the migration to populate fork networks
  Create fork networks for forks for which the source was deleted.
  Fix item name and namespace text overflow in Projects dropdown
  Minor backport from EE
  fix link that was linking to `html` instead of `md`
  Backport epic tasklist
  Add timeouts for Gitaly calls
  SSHUploadPack over Gitaly is now OptOut
  fix icon colors in commit list
  Fix star icon color/stroke
  Backport border inline edit
  Add checkboxes to automatically run AutoDevops pipeline
  BE for automatic pipeline when enabling Auto DevOps
  I am certainly weary of debugging sidekiq but I don't think that's what was meant
  Ensure MRs always use branch refs for comparison
  Fix issue comment submit button disabled on GFM paste
  Lock seed-fu at the correct version in Gemfile.lock
  Improve indexes on merge_request_diffs
  ...

3 files changed, 79 insertions, 37 deletions

diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 3cb9b254e6e..d399636bb28 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -1046,9 +1046,15 @@ module Gitlab
       end
 
       def with_repo_tmp_commit(start_repository, start_branch_name, sha)
+        source_ref = start_branch_name
+
+        unless Gitlab::Git.branch_ref?(source_ref)
+          source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}"
+        end
+
         tmp_ref = fetch_ref(
           start_repository,
-          source_ref: "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}",
+          source_ref: source_ref,
           target_ref: "refs/tmp/#{SecureRandom.hex}"
         )
 
@@ -1058,12 +1064,11 @@ module Gitlab
       end
 
       def fetch_source_branch!(source_repository, source_branch, local_ref)
-        with_repo_branch_commit(source_repository, source_branch) do |commit|
-          if commit
-            write_ref(local_ref, commit.sha)
-            true
+        Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled|
+          if is_enabled
+            gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref)
           else
-            false
+            rugged_fetch_source_branch(source_repository, source_branch, local_ref)
           end
         end
       end
@@ -1151,10 +1156,12 @@ module Gitlab
         @has_visible_content = has_local_branches?
       end
 
-      def fetch(remote = 'origin')
-        args = %W(#{Gitlab.config.git.bin_path} fetch #{remote})
-
-        popen(args, @path).last.zero?
+      # Like all public `Gitlab::Git::Repository` methods, this method is part
+      # of `Repository`'s interface through `method_missing`.
+      # `Repository` has its own `fetch_remote` which uses `gitlab-shell` and
+      # takes some extra attributes, so we qualify this method name to prevent confusion.
+      def fetch_remote_without_shell(remote = 'origin')
+        run_git(['fetch', remote]).last.zero?
       end
 
       def blob_at(sha, path)
@@ -1216,6 +1223,17 @@ module Gitlab
 
       private
 
+      def rugged_fetch_source_branch(source_repository, source_branch, local_ref)
+        with_repo_branch_commit(source_repository, source_branch) do |commit|
+          if commit
+            write_ref(local_ref, commit.sha)
+            true
+          else
+            false
+          end
+        end
+      end
+
       # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'.
       def branches_filter(filter: nil, sort_by: nil)
         # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464
@@ -1233,11 +1251,21 @@ module Gitlab
         sort_branches(branches, sort_by)
       end
 
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695
       def git_merged_branch_names(branch_names = [])
-        lines = run_git(['branch', '--merged', root_ref] + branch_names)
-          .first.lines
+        root_sha = find_branch(root_ref).target
+
+        git_arguments =
+          %W[branch --merged #{root_sha}
+             --format=%(refname:short)\ %(objectname)] + branch_names
 
-        lines.map(&:strip)
+        lines = run_git(git_arguments).first.lines
+
+        lines.each_with_object([]) do |line, branches|
+          name, sha = line.strip.split(' ', 2)
+
+          branches << name if sha != root_sha
+        end
       end
 
       def log_using_shell?(options)
diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb
index 4500482d68f..392bef69e80 100644
--- a/lib/gitlab/git/repository_mirroring.rb
+++ b/lib/gitlab/git/repository_mirroring.rb
@@ -1,38 +1,47 @@
 module Gitlab
   module Git
     module RepositoryMirroring
-      IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze
-      IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze
-      MIRROR_REMOTE = 'mirror'.freeze
+      REFMAPS = {
+        # With `:all_refs`, the repository is equivalent to the result of `git clone --mirror`
+        all_refs: '+refs/*:refs/*',
+        heads: '+refs/heads/*:refs/heads/*',
+        tags: '+refs/tags/*:refs/tags/*'
+      }.freeze
 
       RemoteError = Class.new(StandardError)
 
-      def set_remote_as_mirror(remote_name)
-        # This is used to define repository as equivalent as "git clone --mirror"
-        rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*'
-        rugged.config["remote.#{remote_name}.mirror"] = true
-        rugged.config["remote.#{remote_name}.prune"] = true
-      end
-
-      def set_import_remote_as_mirror(remote_name)
-        # Add first fetch with Rugged so it does not create its own.
-        rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS
-
-        add_remote_fetch_config(remote_name, IMPORT_TAG_REFS)
+      def set_remote_as_mirror(remote_name, refmap: :all_refs)
+        set_remote_refmap(remote_name, refmap)
 
         rugged.config["remote.#{remote_name}.mirror"] = true
         rugged.config["remote.#{remote_name}.prune"] = true
       end
 
-      def add_remote_fetch_config(remote_name, refspec)
-        run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
+      def set_remote_refmap(remote_name, refmap)
+        Array(refmap).each_with_index do |refspec, i|
+          refspec = REFMAPS[refspec] || refspec
+
+          # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it.
+          # To make sure we start from scratch, we set the first using rugged, and use `git` for any others
+          if i == 0
+            rugged.config["remote.#{remote_name}.fetch"] = refspec
+          else
+            run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
+          end
+        end
       end
 
-      def fetch_mirror(url)
-        add_remote(MIRROR_REMOTE, url)
-        set_remote_as_mirror(MIRROR_REMOTE)
-        fetch(MIRROR_REMOTE)
-        remove_remote(MIRROR_REMOTE)
+      # Like all_refs public `Gitlab::Git::Repository` methods, this method is part
+      # of `Repository`'s interface through `method_missing`.
+      # `Repository` has its own `fetch_as_mirror` which uses `gitlab-shell` and
+      # takes some extra attributes, so we qualify this method name to prevent confusion.
+      def fetch_as_mirror_without_shell(url)
+        remote_name = "tmp-#{SecureRandom.hex}"
+        add_remote(remote_name, url)
+        set_remote_as_mirror(remote_name)
+        fetch_remote_without_shell(remote_name)
+      ensure
+        remove_remote(remote_name) if remote_name
       end
 
       def remote_tags(remote)
diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb
index e6b61417de1..e573cd0e143 100644
--- a/lib/gitlab/git/user.rb
+++ b/lib/gitlab/git/user.rb
@@ -8,7 +8,12 @@ module Gitlab
       end
 
       def self.from_gitaly(gitaly_user)
-        new(gitaly_user.gl_username, gitaly_user.name, gitaly_user.email, gitaly_user.gl_id)
+        new(
+          gitaly_user.gl_username,
+          Gitlab::EncodingHelper.encode!(gitaly_user.name),
+          Gitlab::EncodingHelper.encode!(gitaly_user.email),
+          gitaly_user.gl_id
+        )
       end
 
       def initialize(username, name, email, gl_id)
@@ -23,7 +28,7 @@ module Gitlab
       end
 
       def to_gitaly
-        Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name, email: email)
+        Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name.b, email: email.b)
       end
     end
   end