Limit the size of imported GitHub repositorygh-importer-size-limit

This adds a limit on the size of repositories to import from GitHub.
This allows us to prevent users from importing very large repositories,
as doing so could have a negative impact on availability.

3 files changed, 89 insertions, 2 deletions

diff --git a/changelogs/unreleased/gh-importer-size-limit.yml b/changelogs/unreleased/gh-importer-size-limit.yml
new file mode 100644
index 00000000000..c14cbf9c6b4
--- /dev/null
+++ b/changelogs/unreleased/gh-importer-size-limit.yml
@@ -0,0 +1,5 @@
+---
+title: Limit the size of imported GitHub repository
+merge_request:
+author:
+type: performance
diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb
index 01168abde6c..311af9d2a27 100644
--- a/lib/gitlab/github_import/importer/repository_importer.rb
+++ b/lib/gitlab/github_import/importer/repository_importer.rb
@@ -13,18 +13,34 @@ module Gitlab
           @client = client
         end
 
+        def github_repository
+          @github_repository ||= client.repository(project.import_source)
+        end
+
         # Returns true if we should import the wiki for the project.
         def import_wiki?
-          client.repository(project.import_source)&.has_wiki &&
+          github_repository&.has_wiki &&
             !project.wiki_repository_exists? &&
             Gitlab::GitalyClient::RemoteService.exists?(wiki_url)
         end
 
+        def repository_size
+          github_repository.size.to_f
+        end
+
+        def repository_too_large?
+          repository_size > 10_485_760.0 # size is in KB
+        end
+
         # Imports the repository data.
         #
         # This method will return true if the data was imported successfully or
         # the repository had already been imported before.
         def execute
+          if repository_too_large?
+            return fail_import('The repository is too large to import')
+          end
+
           imported =
             # It's possible a repository has already been imported when running
             # this code, e.g. because we had to retry this job after
diff --git a/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb b/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb
index cc9e4b67e72..ed5ab36abff 100644
--- a/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb
+++ b/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb
@@ -14,7 +14,8 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do
       disk_path: 'foo',
       repository: repository,
       create_wiki: true,
-      import_state: import_state
+      import_state: import_state,
+      size: 50
     )
   end
 
@@ -82,6 +83,10 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do
         .and_return(true)
 
       expect(importer)
+        .to receive(:repository_too_large?)
+        .and_return(false)
+
+      expect(importer)
         .to receive(:import_wiki?)
         .and_return(true)
 
@@ -105,6 +110,10 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do
         .and_return(false)
 
       expect(importer)
+        .to receive(:repository_too_large?)
+        .and_return(false)
+
+      expect(importer)
         .to receive(:import_wiki?)
         .and_return(true)
 
@@ -127,6 +136,10 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do
         .and_return(true)
 
       expect(importer)
+        .to receive(:repository_too_large?)
+        .and_return(false)
+
+      expect(importer)
         .to receive(:import_wiki?)
         .and_return(false)
 
@@ -149,6 +162,10 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do
         .and_return(true)
 
       expect(importer)
+        .to receive(:repository_too_large?)
+        .and_return(false)
+
+      expect(importer)
         .to receive(:import_wiki?)
         .and_return(true)
 
@@ -164,6 +181,55 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do
 
       expect(importer.execute).to eq(false)
     end
+
+    context 'when the repository is too large' do
+      it 'does not import the repository' do
+        expect(importer)
+          .to receive(:repository_too_large?)
+          .and_return(true)
+
+        expect(importer)
+          .not_to receive(:import_repository)
+
+        expect(importer)
+          .to receive(:fail_import)
+          .and_return(false)
+
+        expect(importer.execute).to eq(false)
+      end
+    end
+  end
+
+  describe '#repository_size' do
+    it 'returns the size of the repository as a Float' do
+      expect(importer)
+        .to receive(:github_repository)
+        .and_return(repository)
+
+      expect(repository)
+        .to receive(:size)
+        .and_return(10)
+
+      expect(importer.repository_size).to eq(10.0)
+    end
+  end
+
+  describe '#repository_too_large?' do
+    it 'returns true when the repository is too large' do
+      expect(importer)
+        .to receive(:repository_size)
+        .and_return(10_485_7600.0)
+
+      expect(importer.repository_too_large?).to eq(true)
+    end
+
+    it 'returns false when the repository is not too large' do
+      expect(importer)
+        .to receive(:repository_size)
+        .and_return(10.0)
+
+      expect(importer.repository_too_large?).to eq(false)
+    end
   end
 
   describe '#import_repository' do