Fetch GitHub project as a mirror to get all refs at once

4 files changed, 67 insertions, 1 deletions

diff --git a/app/models/concerns/repository_mirroring.rb b/app/models/concerns/repository_mirroring.rb
new file mode 100644
index 00000000000..7aca3f72ae7
--- /dev/null
+++ b/app/models/concerns/repository_mirroring.rb
@@ -0,0 +1,31 @@
+module RepositoryMirroring
+  def storage_path
+    @project.repository_storage_path
+  end
+
+  def add_remote(name, url)
+    raw_repository.remote_add(name, url)
+  rescue Rugged::ConfigError
+    raw_repository.remote_update(name, url: url)
+  end
+
+  def remove_remote(name)
+    raw_repository.remote_delete(name)
+    true
+  rescue Rugged::ConfigError
+    false
+  end
+
+  def set_remote_as_mirror(name)
+    config = raw_repository.rugged.config
+
+    # This is used to define repository as equivalent as "git clone --mirror"
+    config["remote.#{name}.fetch"] = 'refs/*:refs/*'
+    config["remote.#{name}.mirror"] = true
+    config["remote.#{name}.prune"] = true
+  end
+
+  def fetch_remote(remote, forced: false, no_tags: false)
+    gitlab_shell.fetch_remote(storage_path, path_with_namespace, remote, forced: forced, no_tags: no_tags)
+  end
+end
diff --git a/app/models/project.rb b/app/models/project.rb
index f1bba56d32c..83660d8c431 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -551,6 +551,10 @@ class Project < ActiveRecord::Base
     import_type == 'gitea'
   end
 
+  def github_import?
+    import_type == 'github'
+  end
+
   def check_limit
     unless creator.can_create_project? || namespace.kind == 'group'
       projects_limit = creator.projects_limit
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 596650353fc..a9c1ce6782d 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -2,6 +2,7 @@ require 'securerandom'
 
 class Repository
   include Gitlab::ShellAdapter
+  include RepositoryMirroring
 
   attr_accessor :path_with_namespace, :project
 
@@ -1033,6 +1034,13 @@ class Repository
     rugged.references.delete(tmp_ref) if tmp_ref
   end
 
+  def fetch_mirror(remote, url)
+    add_remote(remote, url)
+    set_remote_as_mirror(remote)
+    fetch_remote(remote, forced: true)
+    remove_remote(remote)
+  end
+
   def fetch_ref(source_path, source_ref, target_ref)
     args = %W(#{Gitlab.config.git.bin_path} fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref})
     Gitlab::Popen.popen(args, path_to_repo)
diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb
index d484a96f785..794cc1556a4 100644
--- a/app/services/projects/import_service.rb
+++ b/app/services/projects/import_service.rb
@@ -21,7 +21,11 @@ module Projects
         # In this case, we only want to import issues, not a repository.
         create_repository
       elsif !project.repository_exists?
-        import_repository
+        if project.github_import? || project.gitea_import?
+          fetch_repository
+        else
+          import_repository
+        end
       end
     end
 
@@ -45,6 +49,25 @@ module Projects
       end
     end
 
+    def fetch_repository
+      begin
+        raise Error, 'Blocked import URL.' if Gitlab::UrlBlocker.blocked_url?(project.import_url)
+
+        project.create_repository
+        project.repository.add_remote(project.import_type, project.import_url)
+        project.repository.set_remote_as_mirror (project.import_type)
+        project.repository.fetch_remote(project.import_type, forced: true)
+        project.repository.remove_remote(project.import_type)
+      rescue => e
+        # Expire cache to prevent scenarios such as:
+        # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true
+        # 2. Retried import, repo is broken or not imported but +exists?+ still returns true
+        project.repository.before_import if project.repository_exists?
+
+        raise Error, "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
+      end
+    end
+
     def import_data
       return unless has_importer?