Merge branch 'feature/migrate-bundle-to-disk-to-gitaly' into 'master'

Migrate repository bundling to Gitaly

Closes gitaly#929

See merge request gitlab-org/gitlab-ce!16563

10 files changed, 64 insertions, 10 deletions

diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION
index 7375dee5f49..31b648bd6fa 100644
--- a/GITALY_SERVER_VERSION
+++ b/GITALY_SERVER_VERSION
@@ -1 +1 @@
-0.72.0
+0.73.0
diff --git a/Gemfile b/Gemfile
index a4c3db4bf5f..346182b3852 100644
--- a/Gemfile
+++ b/Gemfile
@@ -406,7 +406,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 0.74.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 0.76.0', require: 'gitaly'
 
 gem 'toml-rb', '~> 0.3.15', require: false
 
diff --git a/Gemfile.lock b/Gemfile.lock
index d69c532b309..1cbeab8d6b5 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -285,7 +285,7 @@ GEM
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
     gherkin-ruby (0.3.2)
-    gitaly-proto (0.74.0)
+    gitaly-proto (0.76.0)
       google-protobuf (~> 3.1)
       grpc (~> 1.0)
     github-linguist (4.7.6)
@@ -1056,7 +1056,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.2.0)
-  gitaly-proto (~> 0.74.0)
+  gitaly-proto (~> 0.76.0)
   github-linguist (~> 4.7.0)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-markup (~> 1.6.2)
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 73c4899cb9b..824e18bec78 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -20,6 +20,7 @@ class Repository
   attr_accessor :full_path, :disk_path, :project, :is_wiki
 
   delegate :ref_name_for_sha, to: :raw_repository
+  delegate :bundle_to_disk, to: :raw_repository
 
   CreateTreeError = Class.new(StandardError)
 
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index d666362de0d..9864e4251f1 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -1268,6 +1268,18 @@ module Gitlab
         success || gitlab_projects_error
       end
 
+      def bundle_to_disk(save_path)
+        gitaly_migrate(:bundle_to_disk) do |is_enabled|
+          if is_enabled
+            gitaly_repository_client.create_bundle(save_path)
+          else
+            run_git!(%W(bundle create #{save_path} --all))
+          end
+        end
+
+        true
+      end
+
       # rubocop:disable Metrics/ParameterLists
       def multi_action(
         user, branch_name:, message:, actions:,
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index 12016aee2a6..654a3c314f1 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -161,6 +161,23 @@ module Gitlab
           return response.error.b, 1
         end
       end
+
+      def create_bundle(save_path)
+        request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo)
+        response = GitalyClient.call(
+          @storage,
+          :repository_service,
+          :create_bundle,
+          request,
+          timeout: GitalyClient.default_timeout
+        )
+
+        File.open(save_path, 'wb') do |f|
+          response.each do |message|
+            f.write(message.data)
+          end
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index dd5d35feab9..25399f307f2 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -11,10 +11,6 @@ module Gitlab
         untar_with_options(archive: archive, dir: dir, options: 'zxf')
       end
 
-      def git_bundle(repo_path:, bundle_path:)
-        execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all))
-      end
-
       def git_clone_bundle(repo_path:, bundle_path:)
         execute(%W(#{git_bin_path} clone --bare -- #{bundle_path} #{repo_path}))
         Gitlab::Git::Repository.create_hooks(repo_path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path))
diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb
index a7028a32570..695462c7dd2 100644
--- a/lib/gitlab/import_export/repo_saver.rb
+++ b/lib/gitlab/import_export/repo_saver.rb
@@ -21,7 +21,7 @@ module Gitlab
 
       def bundle_to_disk
         mkdir_p(@shared.export_path)
-        git_bundle(repo_path: path_to_repo, bundle_path: @full_path)
+        @project.repository.bundle_to_disk(@full_path)
       rescue => e
         @shared.error(e)
         false
diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb
index 1e6722a7bba..5fa2e101e29 100644
--- a/lib/gitlab/import_export/wiki_repo_saver.rb
+++ b/lib/gitlab/import_export/wiki_repo_saver.rb
@@ -10,7 +10,7 @@ module Gitlab
 
       def bundle_to_disk(full_path)
         mkdir_p(@shared.export_path)
-        git_bundle(repo_path: path_to_repo, bundle_path: full_path)
+        @wiki.repository.bundle_to_disk(full_path)
       rescue => e
         @shared.error(e)
         false
diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb
index aec7cde6df8..36ca3980de9 100644
--- a/spec/lib/gitlab/git/repository_spec.rb
+++ b/spec/lib/gitlab/git/repository_spec.rb
@@ -1926,6 +1926,34 @@ describe Gitlab::Git::Repository, seed_helper: true do
     it { expect(subject.repository_relative_path).to eq(repository.relative_path) }
   end
 
+  describe '#bundle_to_disk' do
+    shared_examples 'bundling to disk' do
+      let(:save_path) { File.join(Dir.tmpdir, "repo-#{SecureRandom.hex}.bundle") }
+
+      after do
+        FileUtils.rm_rf(save_path)
+      end
+
+      it 'saves a bundle to disk' do
+        repository.bundle_to_disk(save_path)
+
+        success = system(
+          *%W(#{Gitlab.config.git.bin_path} -C #{repository.path} bundle verify #{save_path}),
+          [:out, :err] => '/dev/null'
+        )
+        expect(success).to be true
+      end
+    end
+
+    context 'when Gitaly bundle_to_disk feature is enabled' do
+      it_behaves_like 'bundling to disk'
+    end
+
+    context 'when Gitaly bundle_to_disk feature is disabled', :disable_gitaly do
+      it_behaves_like 'bundling to disk'
+    end
+  end
+
   context 'gitlab_projects commands' do
     let(:gitlab_projects) { repository.gitlab_projects }
     let(:timeout) { Gitlab.config.gitlab_shell.git_timeout }