Clean up repository fetch and mirror methodsdm-cleanup-fetch-and-mirror-methods

9 files changed, 73 insertions, 115 deletions

diff --git a/app/models/repository.rb b/app/models/repository.rb
index 3a89fa9264b..be6fd7a77a6 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -965,6 +965,19 @@ class Repository
     run_git(args).first.lines.map(&:strip)
   end
 
+  def fetch_as_mirror(url, forced: false, fetch_refs: :all, remote_name: nil)
+    unless remote_name
+      remote_name = "tmp-#{SecureRandom.hex}"
+      tmp_remote_name = true
+    end
+
+    add_remote(remote_name, url)
+    set_remote_as_mirror(remote_name, fetch_refs: fetch_refs)
+    fetch_remote(remote_name, forced: forced)
+  ensure
+    remove_remote(remote_name) if tmp_remote_name
+  end
+
   def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false)
     gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
   end
@@ -1066,6 +1079,10 @@ class Repository
     raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref)
   end
 
+  def repository_storage_path
+    @project.repository_storage_path
+  end
+
   private
 
   # TODO Generice finder, later split this on finders by Ref or Oid
@@ -1131,10 +1148,6 @@ class Repository
     raw_repository.run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip
   end
 
-  def repository_storage_path
-    @project.repository_storage_path
-  end
-
   def initialize_raw_repository
     Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, is_wiki))
   end
diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb
index c3b11341b4d..26248eb8030 100644
--- a/app/services/projects/import_service.rb
+++ b/app/services/projects/import_service.rb
@@ -51,10 +51,13 @@ module Projects
 
     def import_repository
       begin
-        if project.gitea_import?
-          fetch_repository
+        fetch_refs = importer_class.try(:fetch_refs) if has_importer?
+
+        if fetch_refs
+          project.ensure_repository
+          project.repository.fetch_as_mirror(project.import_url, fetch_refs: fetch_refs)
         else
-          clone_repository
+          gitlab_shell.import_repository(project.repository_storage_path, project.disk_path, project.import_url)
         end
       rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e
         # Expire cache to prevent scenarios such as:
@@ -66,17 +69,6 @@ module Projects
       end
     end
 
-    def clone_repository
-      gitlab_shell.import_repository(project.repository_storage_path, project.disk_path, project.import_url)
-    end
-
-    def fetch_repository
-      project.ensure_repository
-      project.repository.add_remote(project.import_type, project.import_url)
-      project.repository.set_remote_as_mirror(project.import_type)
-      project.repository.fetch_remote(project.import_type, forced: true)
-    end
-
     def import_data
       return unless has_importer?
 
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index cfb88a0c12b..19008380353 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -1141,10 +1141,12 @@ module Gitlab
         @has_visible_content = has_local_branches?
       end
 
-      def fetch(remote = 'origin')
-        args = %W(#{Gitlab.config.git.bin_path} fetch #{remote})
-
-        popen(args, @path).last.zero?
+      # Like all public `Gitlab::Git::Repository` methods, this method is part
+      # of `Repository`'s interface through `method_missing`.
+      # `Repository` has its own `fetch_remote` which uses `gitlab-shell` and
+      # takes some extra attributes, so we qualify this method name to prevent confusion.
+      def fetch_remote_without_shell(remote = 'origin')
+        run_git(['fetch', remote]).last.zero?
       end
 
       def blob_at(sha, path)
diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb
index 637e7a0659c..d259c102466 100644
--- a/lib/gitlab/git/repository_mirroring.rb
+++ b/lib/gitlab/git/repository_mirroring.rb
@@ -1,24 +1,26 @@
 module Gitlab
   module Git
     module RepositoryMirroring
-      IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze
-      IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze
-      MIRROR_REMOTE = 'mirror'.freeze
+      FETCH_REFS = {
+        # `:all` is used to define repository as equivalent as "git clone --mirror"
+        all: '+refs/*:refs/*',
+        heads: '+refs/heads/*:refs/heads/*',
+        tags: '+refs/tags/*:refs/tags/*'
+      }.freeze
 
       RemoteError = Class.new(StandardError)
 
-      def set_remote_as_mirror(remote_name)
-        # This is used to define repository as equivalent as "git clone --mirror"
-        rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*'
-        rugged.config["remote.#{remote_name}.mirror"] = true
-        rugged.config["remote.#{remote_name}.prune"] = true
-      end
+      def set_remote_as_mirror(remote_name, fetch_refs: :all)
+        Array(fetch_refs).each_with_index do |fetch_ref, i|
+          fetch_ref = FETCH_REFS[fetch_ref] || fetch_ref
 
-      def set_import_remote_as_mirror(remote_name)
-        # Add first fetch with Rugged so it does not create its own.
-        rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS
-
-        add_remote_fetch_config(remote_name, IMPORT_TAG_REFS)
+          # Add first fetch with Rugged so it does not create its own.
+          if i == 0
+            rugged.config["remote.#{remote_name}.fetch"] = fetch_ref
+          else
+            add_remote_fetch_config(remote_name, fetch_ref)
+          end
+        end
 
         rugged.config["remote.#{remote_name}.mirror"] = true
         rugged.config["remote.#{remote_name}.prune"] = true
@@ -28,11 +30,17 @@ module Gitlab
         run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
       end
 
-      def fetch_mirror(url)
-        add_remote(MIRROR_REMOTE, url)
-        set_remote_as_mirror(MIRROR_REMOTE)
-        fetch(MIRROR_REMOTE)
-        remove_remote(MIRROR_REMOTE)
+      # Like all public `Gitlab::Git::Repository` methods, this method is part
+      # of `Repository`'s interface through `method_missing`.
+      # `Repository` has its own `fetch_as_mirror` which uses `gitlab-shell` and
+      # takes some extra attributes, so we qualify this method name to prevent confusion.
+      def fetch_as_mirror_without_shell(url)
+        remote_name = "tmp-#{SecureRandom.hex}"
+        add_remote(remote_name, url)
+        set_remote_as_mirror(remote_name)
+        fetch_remote_without_shell(remote_name)
+      ensure
+        remove_remote(remote_name) if remote_name
       end
 
       def remote_tags(remote)
diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb
index d2ae4c1255e..427dab82e19 100644
--- a/lib/gitlab/github_import.rb
+++ b/lib/gitlab/github_import.rb
@@ -1,5 +1,9 @@
 module Gitlab
   module GithubImport
+    def self.fetch_refs
+      [:heads, :tags, '+refs/pull/*/head:refs/merge-requests/*/head']
+    end
+
     def self.new_client_for(project, token: nil, parallel: true)
       token_to_use = token || project.import_data&.credentials&.fetch(:user)
 
diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb
index 0b67fc8db73..7b1994de012 100644
--- a/lib/gitlab/github_import/importer/repository_importer.rb
+++ b/lib/gitlab/github_import/importer/repository_importer.rb
@@ -45,27 +45,14 @@ module Gitlab
         def import_repository
           project.ensure_repository
 
-          configure_repository_remote
-
-          project.repository.fetch_remote('github', forced: true)
+          fetch_refs = Gitlab::GithubImport.fetch_refs
+          project.repository.fetch_as_mirror(project.import_url, fetch_refs: fetch_refs, forced: true, remote_name: 'github')
 
           true
         rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e
           fail_import("Failed to import the repository: #{e.message}")
         end
 
-        def configure_repository_remote
-          return if project.repository.remote_exists?('github')
-
-          project.repository.add_remote('github', project.import_url)
-          project.repository.set_import_remote_as_mirror('github')
-
-          project.repository.add_remote_fetch_config(
-            'github',
-            '+refs/pull/*/head:refs/merge-requests/*/head'
-          )
-        end
-
         def import_wiki_repository
           wiki_path = "#{project.disk_path}.wiki"
           wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git')
diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb
index 12c968805f5..28ad65d2e7a 100644
--- a/lib/gitlab/legacy_github_import/importer.rb
+++ b/lib/gitlab/legacy_github_import/importer.rb
@@ -3,6 +3,10 @@ module Gitlab
     class Importer
       include Gitlab::ShellAdapter
 
+      def self.fetch_refs
+        Gitlab::GithubImport.fetch_refs
+      end
+
       attr_reader :errors, :project, :repo, :repo_url
 
       def initialize(project)
diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb
index 5d990b42c24..9b3117fb387 100644
--- a/spec/lib/gitlab/git/repository_spec.rb
+++ b/spec/lib/gitlab/git/repository_spec.rb
@@ -588,12 +588,12 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
-  describe '#fetch_mirror' do
+  describe '#fetch_as_mirror_without_shell' do
     let(:new_repository) do
       Gitlab::Git::Repository.new('default', 'my_project.git', '')
     end
 
-    subject { new_repository.fetch_mirror(repository.path) }
+    subject { new_repository.fetch_as_mirror_without_shell(repository.path) }
 
     before do
       Gitlab::Shell.new.add_repository('default', 'my_project')
@@ -1630,15 +1630,15 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
-  describe '#fetch' do
+  describe '#fetch_remote_without_shell' do
     let(:git_path) { Gitlab.config.git.bin_path }
     let(:remote_name) { 'my_remote' }
 
-    subject { repository.fetch(remote_name) }
+    subject { repository.fetch_remote_without_shell(remote_name) }
 
     it 'fetches the remote and returns true if the command was successful' do
       expect(repository).to receive(:popen)
-        .with(%W(#{git_path} fetch #{remote_name}), repository.path)
+        .with(%W(#{git_path} fetch #{remote_name}), repository.path, {})
         .and_return(['', 0])
 
       expect(subject).to be(true)
@@ -1755,21 +1755,6 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
-  describe '#fetch' do
-    let(:git_path) { Gitlab.config.git.bin_path }
-    let(:remote_name) { 'my_remote' }
-
-    subject { repository.fetch(remote_name) }
-
-    it 'fetches the remote and returns true if the command was successful' do
-      expect(repository).to receive(:popen)
-        .with(%W(#{git_path} fetch #{remote_name}), repository.path)
-        .and_return(['', 0])
-
-      expect(subject).to be(true)
-    end
-  end
-
   def create_remote_branch(repository, remote_name, branch_name, source_branch_name)
     source_branch = repository.branches.find { |branch| branch.name == source_branch_name }
     rugged = repository.rugged
diff --git a/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb b/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb
index 80539807711..f827902e7b8 100644
--- a/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb
+++ b/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb
@@ -164,12 +164,9 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do
       expect(project)
         .to receive(:ensure_repository)
 
-      expect(importer)
-        .to receive(:configure_repository_remote)
-
       expect(repository)
-        .to receive(:fetch_remote)
-        .with('github', forced: true)
+        .to receive(:fetch_as_mirror)
+        .with(project.import_url, fetch_refs: Gitlab::GithubImport.fetch_refs, forced: true, remote_name: 'github')
 
       expect(importer.import_repository).to eq(true)
     end
@@ -186,40 +183,6 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do
     end
   end
 
-  describe '#configure_repository_remote' do
-    it 'configures the remote details' do
-      expect(repository)
-        .to receive(:remote_exists?)
-        .with('github')
-        .and_return(false)
-
-      expect(repository)
-        .to receive(:add_remote)
-        .with('github', 'foo.git')
-
-      expect(repository)
-        .to receive(:set_import_remote_as_mirror)
-        .with('github')
-
-      expect(repository)
-        .to receive(:add_remote_fetch_config)
-
-      importer.configure_repository_remote
-    end
-
-    it 'does not configure the remote if already configured' do
-      expect(repository)
-        .to receive(:remote_exists?)
-        .with('github')
-        .and_return(true)
-
-      expect(repository)
-        .not_to receive(:add_remote)
-
-      importer.configure_repository_remote
-    end
-  end
-
   describe '#import_wiki_repository' do
     it 'imports the wiki repository' do
       expect(importer.gitlab_shell)