Add client methods for FetchIntoObjectPool RPC

Gitaly's FetchIntoObjectPool RPC will idempotently fetch objects into an
object pool. If the pool doesn't exist, it will create an empty pool
before attempting the fetch. This change adds client code as well as
specs to cover this behavior.

8 files changed, 156 insertions, 80 deletions

diff --git a/Gemfile b/Gemfile
index 6654b285a72..bbc777fec9e 100644
--- a/Gemfile
+++ b/Gemfile
@@ -417,7 +417,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 1.22.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 1.26.0', require: 'gitaly'
 
 gem 'grpc', '~> 1.19.0'
 
diff --git a/Gemfile.lock b/Gemfile.lock
index ceece1da8d7..0ee59793469 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -283,7 +283,7 @@ GEM
       gettext_i18n_rails (>= 0.7.1)
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
-    gitaly-proto (1.22.0)
+    gitaly-proto (1.26.0)
       grpc (~> 1.0)
     github-markup (1.7.0)
     gitlab-default_value_for (3.1.1)
@@ -1056,7 +1056,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.3)
-  gitaly-proto (~> 1.22.0)
+  gitaly-proto (~> 1.26.0)
   github-markup (~> 1.7.0)
   gitlab-default_value_for (~> 3.1.1)
   gitlab-labkit (~> 0.1.2)
diff --git a/lib/gitlab/git/object_pool.rb b/lib/gitlab/git/object_pool.rb
index 8eb3c28ab70..d0577d7a4ff 100644
--- a/lib/gitlab/git/object_pool.rb
+++ b/lib/gitlab/git/object_pool.rb
@@ -40,6 +40,10 @@ module Gitlab
         @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY, gl_project_path)
       end
 
+      def fetch
+        object_pool_service.fetch(source_repository)
+      end
+
       private
 
       def object_pool_service
diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb
index ce1fb4d68ae..d7fac26bc13 100644
--- a/lib/gitlab/gitaly_client/object_pool_service.rb
+++ b/lib/gitlab/gitaly_client/object_pool_service.rb
@@ -33,6 +33,15 @@ module Gitlab
         GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool,
                           request, timeout: GitalyClient.fast_timeout)
       end
+
+      def fetch(repository)
+        request = Gitaly::FetchIntoObjectPoolRequest.new(
+          object_pool: object_pool,
+          origin: repository.gitaly_repository
+        )
+
+        GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request)
+      end
     end
   end
 end
diff --git a/spec/lib/gitlab/git/object_pool_spec.rb b/spec/lib/gitlab/git/object_pool_spec.rb
index 0d5069568e1..6511c2b61bf 100644
--- a/spec/lib/gitlab/git/object_pool_spec.rb
+++ b/spec/lib/gitlab/git/object_pool_spec.rb
@@ -3,8 +3,12 @@
 require 'spec_helper'
 
 describe Gitlab::Git::ObjectPool do
+  include RepoHelpers
+
   let(:pool_repository) { create(:pool_repository) }
   let(:source_repository) { pool_repository.source_project.repository }
+  let(:source_repository_path) { File.join(TestEnv.repos_path, source_repository.relative_path) }
+  let(:source_repository_rugged) { Rugged::Repository.new(source_repository_path) }
 
   subject { pool_repository.object_pool }
 
@@ -76,4 +80,41 @@ describe Gitlab::Git::ObjectPool do
       end
     end
   end
+
+  describe '#fetch' do
+    let(:commit_count) { source_repository.commit_count }
+
+    context "when the object's pool repository exists" do
+      it 'does not raise an error' do
+        expect { subject.fetch }.not_to raise_error
+      end
+    end
+
+    context "when the object's pool repository does not exist" do
+      before do
+        subject.delete
+      end
+
+      it "re-creates the object pool's repository" do
+        subject.fetch
+
+        expect(subject.repository.exists?).to be(true)
+      end
+
+      it 'does not raise an error' do
+        expect { subject.fetch }.not_to raise_error
+      end
+
+      it 'fetches objects from the source repository' do
+        new_commit_id = new_commit_edit_old_file(source_repository_rugged).oid
+
+        expect(subject.repository.exists?).to be false
+
+        subject.fetch
+
+        expect(subject.repository.commit_count('refs/remotes/origin/master')).to eq(commit_count)
+        expect(subject.repository.commit(new_commit_id).id).to eq(new_commit_id)
+      end
+    end
+  end
 end
diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb
index 5f8a2848944..0f6aac9b6de 100644
--- a/spec/lib/gitlab/git/repository_spec.rb
+++ b/spec/lib/gitlab/git/repository_spec.rb
@@ -3,6 +3,7 @@ require "spec_helper"
 
 describe Gitlab::Git::Repository, :seed_helper do
   include Gitlab::EncodingHelper
+  include RepoHelpers
   using RSpec::Parameterized::TableSyntax
 
   shared_examples 'wrapping gRPC errors' do |gitaly_client_class, gitaly_client_method|
@@ -2209,83 +2210,6 @@ describe Gitlab::Git::Repository, :seed_helper do
     repository_rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", source_branch.dereferenced_target.sha)
   end
 
-  # Build the options hash that's passed to Rugged::Commit#create
-  def commit_options(repo, index, target, ref, message)
-    options = {}
-    options[:tree] = index.write_tree(repo)
-    options[:author] = {
-      email: "test@example.com",
-      name: "Test Author",
-      time: Time.gm(2014, "mar", 3, 20, 15, 1)
-    }
-    options[:committer] = {
-      email: "test@example.com",
-      name: "Test Author",
-      time: Time.gm(2014, "mar", 3, 20, 15, 1)
-    }
-    options[:message] ||= message
-    options[:parents] = repo.empty? ? [] : [target].compact
-    options[:update_ref] = ref
-
-    options
-  end
-
-  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
-  # contents of CHANGELOG with a single new line of text.
-  def new_commit_edit_old_file(repo)
-    oid = repo.write("I replaced the changelog with this text", :blob)
-    index = repo.index
-    index.read_tree(repo.head.target.tree)
-    index.add(path: "CHANGELOG", oid: oid, mode: 0100644)
-
-    options = commit_options(
-      repo,
-      index,
-      repo.head.target,
-      "HEAD",
-      "Edit CHANGELOG in its original location"
-    )
-
-    sha = Rugged::Commit.create(repo, options)
-    repo.lookup(sha)
-  end
-
-  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
-  # contents of the specified file_path with new text.
-  def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head)
-    oid = repo.write(text, :blob)
-    index = repo.index
-    index.read_tree(branch.target.tree)
-    index.add(path: file_path, oid: oid, mode: 0100644)
-    options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message)
-    sha = Rugged::Commit.create(repo, options)
-    repo.lookup(sha)
-  end
-
-  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
-  # contents of encoding/CHANGELOG with new text.
-  def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text)
-    branch = repo.branches[branch_name]
-    new_commit_edit_new_file(repo, file_path, commit_message, text, branch)
-  end
-
-  # Writes a new commit to the repo and returns a Rugged::Commit.  Moves the
-  # CHANGELOG file to the encoding/ directory.
-  def new_commit_move_file(repo)
-    blob_oid = repo.head.target.tree.detect { |i| i[:name] == "CHANGELOG" }[:oid]
-    file_content = repo.lookup(blob_oid).content
-    oid = repo.write(file_content, :blob)
-    index = repo.index
-    index.read_tree(repo.head.target.tree)
-    index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644)
-    index.remove("CHANGELOG")
-
-    options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/")
-
-    sha = Rugged::Commit.create(repo, options)
-    repo.lookup(sha)
-  end
-
   def refs(dir)
     IO.popen(%W[git -C #{dir} for-each-ref], &:read).split("\n").map do |line|
       line.split("\t").last
diff --git a/spec/lib/gitlab/gitaly_client/object_pool_service_spec.rb b/spec/lib/gitlab/gitaly_client/object_pool_service_spec.rb
index 149b7ec5bb0..0e0c3d329b5 100644
--- a/spec/lib/gitlab/gitaly_client/object_pool_service_spec.rb
+++ b/spec/lib/gitlab/gitaly_client/object_pool_service_spec.rb
@@ -43,4 +43,24 @@ describe Gitlab::GitalyClient::ObjectPoolService do
       end
     end
   end
+
+  describe '#fetch' do
+    before do
+      subject.delete
+    end
+
+    it 'restores the pool repository objects' do
+      subject.fetch(project.repository)
+
+      expect(object_pool.repository.exists?).to be(true)
+    end
+
+    context 'when called twice' do
+      it "doesn't raise an error" do
+        subject.delete
+
+        expect { subject.fetch(project.repository) }.not_to raise_error
+      end
+    end
+  end
 end
diff --git a/spec/support/helpers/repo_helpers.rb b/spec/support/helpers/repo_helpers.rb
index 4af90f4af79..44d95a029af 100644
--- a/spec/support/helpers/repo_helpers.rb
+++ b/spec/support/helpers/repo_helpers.rb
@@ -11,6 +11,8 @@ module RepoHelpers
   #   blob.path # => 'files/js/commit.js.coffee'
   #   blob.data # => 'class Commit...'
   #
+  # Build the options hash that's passed to Rugged::Commit#create
+
   def sample_blob
     OpenStruct.new(
       oid: '5f53439ca4b009096571d3c8bc3d09d30e7431b3',
@@ -129,4 +131,80 @@ eos
       file_content: content
     ).execute
   end
+
+  def commit_options(repo, index, target, ref, message)
+    options = {}
+    options[:tree] = index.write_tree(repo)
+    options[:author] = {
+      email: "test@example.com",
+      name: "Test Author",
+      time: Time.gm(2014, "mar", 3, 20, 15, 1)
+    }
+    options[:committer] = {
+      email: "test@example.com",
+      name: "Test Author",
+      time: Time.gm(2014, "mar", 3, 20, 15, 1)
+    }
+    options[:message] ||= message
+    options[:parents] = repo.empty? ? [] : [target].compact
+    options[:update_ref] = ref
+
+    options
+  end
+
+  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
+  # contents of CHANGELOG with a single new line of text.
+  def new_commit_edit_old_file(repo)
+    oid = repo.write("I replaced the changelog with this text", :blob)
+    index = repo.index
+    index.read_tree(repo.head.target.tree)
+    index.add(path: "CHANGELOG", oid: oid, mode: 0100644)
+
+    options = commit_options(
+      repo,
+      index,
+      repo.head.target,
+      "HEAD",
+      "Edit CHANGELOG in its original location"
+    )
+
+    sha = Rugged::Commit.create(repo, options)
+    repo.lookup(sha)
+  end
+
+  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
+  # contents of the specified file_path with new text.
+  def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head)
+    oid = repo.write(text, :blob)
+    index = repo.index
+    index.read_tree(branch.target.tree)
+    index.add(path: file_path, oid: oid, mode: 0100644)
+    options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message)
+    sha = Rugged::Commit.create(repo, options)
+    repo.lookup(sha)
+  end
+
+  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
+  # contents of encoding/CHANGELOG with new text.
+  def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text)
+    branch = repo.branches[branch_name]
+    new_commit_edit_new_file(repo, file_path, commit_message, text, branch)
+  end
+
+  # Writes a new commit to the repo and returns a Rugged::Commit.  Moves the
+  # CHANGELOG file to the encoding/ directory.
+  def new_commit_move_file(repo)
+    blob_oid = repo.head.target.tree.detect { |i| i[:name] == "CHANGELOG" }[:oid]
+    file_content = repo.lookup(blob_oid).content
+    oid = repo.write(file_content, :blob)
+    index = repo.index
+    index.read_tree(repo.head.target.tree)
+    index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644)
+    index.remove("CHANGELOG")
+
+    options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/")
+
+    sha = Rugged::Commit.create(repo, options)
+    repo.lookup(sha)
+  end
 end