From 5ee7876534891df9c2f5ab6c5112cc47f98b1df5 Mon Sep 17 00:00:00 2001 From: John Cai Date: Wed, 24 Apr 2019 18:50:45 -0700 Subject: Add client methods for FetchIntoObjectPool RPC Gitaly's FetchIntoObjectPool RPC will idempotently fetch objects into an object pool. If the pool doesn't exist, it will create an empty pool before attempting the fetch. This change adds client code as well as specs to cover this behavior. --- Gemfile | 2 +- Gemfile.lock | 4 +- lib/gitlab/git/object_pool.rb | 4 ++ lib/gitlab/gitaly_client/object_pool_service.rb | 9 +++ spec/lib/gitlab/git/object_pool_spec.rb | 41 ++++++++++++ spec/lib/gitlab/git/repository_spec.rb | 78 +--------------------- .../gitaly_client/object_pool_service_spec.rb | 20 ++++++ spec/support/helpers/repo_helpers.rb | 78 ++++++++++++++++++++++ 8 files changed, 156 insertions(+), 80 deletions(-) diff --git a/Gemfile b/Gemfile index 6654b285a72..bbc777fec9e 100644 --- a/Gemfile +++ b/Gemfile @@ -417,7 +417,7 @@ group :ed25519 do end # Gitaly GRPC client -gem 'gitaly-proto', '~> 1.22.0', require: 'gitaly' +gem 'gitaly-proto', '~> 1.26.0', require: 'gitaly' gem 'grpc', '~> 1.19.0' diff --git a/Gemfile.lock b/Gemfile.lock index ceece1da8d7..0ee59793469 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -283,7 +283,7 @@ GEM gettext_i18n_rails (>= 0.7.1) po_to_json (>= 1.0.0) rails (>= 3.2.0) - gitaly-proto (1.22.0) + gitaly-proto (1.26.0) grpc (~> 1.0) github-markup (1.7.0) gitlab-default_value_for (3.1.1) @@ -1056,7 +1056,7 @@ DEPENDENCIES gettext (~> 3.2.2) gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails_js (~> 1.3) - gitaly-proto (~> 1.22.0) + gitaly-proto (~> 1.26.0) github-markup (~> 1.7.0) gitlab-default_value_for (~> 3.1.1) gitlab-labkit (~> 0.1.2) diff --git a/lib/gitlab/git/object_pool.rb b/lib/gitlab/git/object_pool.rb index 8eb3c28ab70..d0577d7a4ff 100644 --- a/lib/gitlab/git/object_pool.rb +++ b/lib/gitlab/git/object_pool.rb @@ -40,6 +40,10 @@ module Gitlab @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY, gl_project_path) end + def fetch + object_pool_service.fetch(source_repository) + end + private def object_pool_service diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb index ce1fb4d68ae..d7fac26bc13 100644 --- a/lib/gitlab/gitaly_client/object_pool_service.rb +++ b/lib/gitlab/gitaly_client/object_pool_service.rb @@ -33,6 +33,15 @@ module Gitlab GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool, request, timeout: GitalyClient.fast_timeout) end + + def fetch(repository) + request = Gitaly::FetchIntoObjectPoolRequest.new( + object_pool: object_pool, + origin: repository.gitaly_repository + ) + + GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request) + end end end end diff --git a/spec/lib/gitlab/git/object_pool_spec.rb b/spec/lib/gitlab/git/object_pool_spec.rb index 0d5069568e1..6511c2b61bf 100644 --- a/spec/lib/gitlab/git/object_pool_spec.rb +++ b/spec/lib/gitlab/git/object_pool_spec.rb @@ -3,8 +3,12 @@ require 'spec_helper' describe Gitlab::Git::ObjectPool do + include RepoHelpers + let(:pool_repository) { create(:pool_repository) } let(:source_repository) { pool_repository.source_project.repository } + let(:source_repository_path) { File.join(TestEnv.repos_path, source_repository.relative_path) } + let(:source_repository_rugged) { Rugged::Repository.new(source_repository_path) } subject { pool_repository.object_pool } @@ -76,4 +80,41 @@ describe Gitlab::Git::ObjectPool do end end end + + describe '#fetch' do + let(:commit_count) { source_repository.commit_count } + + context "when the object's pool repository exists" do + it 'does not raise an error' do + expect { subject.fetch }.not_to raise_error + end + end + + context "when the object's pool repository does not exist" do + before do + subject.delete + end + + it "re-creates the object pool's repository" do + subject.fetch + + expect(subject.repository.exists?).to be(true) + end + + it 'does not raise an error' do + expect { subject.fetch }.not_to raise_error + end + + it 'fetches objects from the source repository' do + new_commit_id = new_commit_edit_old_file(source_repository_rugged).oid + + expect(subject.repository.exists?).to be false + + subject.fetch + + expect(subject.repository.commit_count('refs/remotes/origin/master')).to eq(commit_count) + expect(subject.repository.commit(new_commit_id).id).to eq(new_commit_id) + end + end + end end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 5f8a2848944..0f6aac9b6de 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -3,6 +3,7 @@ require "spec_helper" describe Gitlab::Git::Repository, :seed_helper do include Gitlab::EncodingHelper + include RepoHelpers using RSpec::Parameterized::TableSyntax shared_examples 'wrapping gRPC errors' do |gitaly_client_class, gitaly_client_method| @@ -2209,83 +2210,6 @@ describe Gitlab::Git::Repository, :seed_helper do repository_rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", source_branch.dereferenced_target.sha) end - # Build the options hash that's passed to Rugged::Commit#create - def commit_options(repo, index, target, ref, message) - options = {} - options[:tree] = index.write_tree(repo) - options[:author] = { - email: "test@example.com", - name: "Test Author", - time: Time.gm(2014, "mar", 3, 20, 15, 1) - } - options[:committer] = { - email: "test@example.com", - name: "Test Author", - time: Time.gm(2014, "mar", 3, 20, 15, 1) - } - options[:message] ||= message - options[:parents] = repo.empty? ? [] : [target].compact - options[:update_ref] = ref - - options - end - - # Writes a new commit to the repo and returns a Rugged::Commit. Replaces the - # contents of CHANGELOG with a single new line of text. - def new_commit_edit_old_file(repo) - oid = repo.write("I replaced the changelog with this text", :blob) - index = repo.index - index.read_tree(repo.head.target.tree) - index.add(path: "CHANGELOG", oid: oid, mode: 0100644) - - options = commit_options( - repo, - index, - repo.head.target, - "HEAD", - "Edit CHANGELOG in its original location" - ) - - sha = Rugged::Commit.create(repo, options) - repo.lookup(sha) - end - - # Writes a new commit to the repo and returns a Rugged::Commit. Replaces the - # contents of the specified file_path with new text. - def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head) - oid = repo.write(text, :blob) - index = repo.index - index.read_tree(branch.target.tree) - index.add(path: file_path, oid: oid, mode: 0100644) - options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message) - sha = Rugged::Commit.create(repo, options) - repo.lookup(sha) - end - - # Writes a new commit to the repo and returns a Rugged::Commit. Replaces the - # contents of encoding/CHANGELOG with new text. - def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text) - branch = repo.branches[branch_name] - new_commit_edit_new_file(repo, file_path, commit_message, text, branch) - end - - # Writes a new commit to the repo and returns a Rugged::Commit. Moves the - # CHANGELOG file to the encoding/ directory. - def new_commit_move_file(repo) - blob_oid = repo.head.target.tree.detect { |i| i[:name] == "CHANGELOG" }[:oid] - file_content = repo.lookup(blob_oid).content - oid = repo.write(file_content, :blob) - index = repo.index - index.read_tree(repo.head.target.tree) - index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644) - index.remove("CHANGELOG") - - options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/") - - sha = Rugged::Commit.create(repo, options) - repo.lookup(sha) - end - def refs(dir) IO.popen(%W[git -C #{dir} for-each-ref], &:read).split("\n").map do |line| line.split("\t").last diff --git a/spec/lib/gitlab/gitaly_client/object_pool_service_spec.rb b/spec/lib/gitlab/gitaly_client/object_pool_service_spec.rb index 149b7ec5bb0..0e0c3d329b5 100644 --- a/spec/lib/gitlab/gitaly_client/object_pool_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/object_pool_service_spec.rb @@ -43,4 +43,24 @@ describe Gitlab::GitalyClient::ObjectPoolService do end end end + + describe '#fetch' do + before do + subject.delete + end + + it 'restores the pool repository objects' do + subject.fetch(project.repository) + + expect(object_pool.repository.exists?).to be(true) + end + + context 'when called twice' do + it "doesn't raise an error" do + subject.delete + + expect { subject.fetch(project.repository) }.not_to raise_error + end + end + end end diff --git a/spec/support/helpers/repo_helpers.rb b/spec/support/helpers/repo_helpers.rb index 4af90f4af79..44d95a029af 100644 --- a/spec/support/helpers/repo_helpers.rb +++ b/spec/support/helpers/repo_helpers.rb @@ -11,6 +11,8 @@ module RepoHelpers # blob.path # => 'files/js/commit.js.coffee' # blob.data # => 'class Commit...' # + # Build the options hash that's passed to Rugged::Commit#create + def sample_blob OpenStruct.new( oid: '5f53439ca4b009096571d3c8bc3d09d30e7431b3', @@ -129,4 +131,80 @@ eos file_content: content ).execute end + + def commit_options(repo, index, target, ref, message) + options = {} + options[:tree] = index.write_tree(repo) + options[:author] = { + email: "test@example.com", + name: "Test Author", + time: Time.gm(2014, "mar", 3, 20, 15, 1) + } + options[:committer] = { + email: "test@example.com", + name: "Test Author", + time: Time.gm(2014, "mar", 3, 20, 15, 1) + } + options[:message] ||= message + options[:parents] = repo.empty? ? [] : [target].compact + options[:update_ref] = ref + + options + end + + # Writes a new commit to the repo and returns a Rugged::Commit. Replaces the + # contents of CHANGELOG with a single new line of text. + def new_commit_edit_old_file(repo) + oid = repo.write("I replaced the changelog with this text", :blob) + index = repo.index + index.read_tree(repo.head.target.tree) + index.add(path: "CHANGELOG", oid: oid, mode: 0100644) + + options = commit_options( + repo, + index, + repo.head.target, + "HEAD", + "Edit CHANGELOG in its original location" + ) + + sha = Rugged::Commit.create(repo, options) + repo.lookup(sha) + end + + # Writes a new commit to the repo and returns a Rugged::Commit. Replaces the + # contents of the specified file_path with new text. + def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head) + oid = repo.write(text, :blob) + index = repo.index + index.read_tree(branch.target.tree) + index.add(path: file_path, oid: oid, mode: 0100644) + options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message) + sha = Rugged::Commit.create(repo, options) + repo.lookup(sha) + end + + # Writes a new commit to the repo and returns a Rugged::Commit. Replaces the + # contents of encoding/CHANGELOG with new text. + def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text) + branch = repo.branches[branch_name] + new_commit_edit_new_file(repo, file_path, commit_message, text, branch) + end + + # Writes a new commit to the repo and returns a Rugged::Commit. Moves the + # CHANGELOG file to the encoding/ directory. + def new_commit_move_file(repo) + blob_oid = repo.head.target.tree.detect { |i| i[:name] == "CHANGELOG" }[:oid] + file_content = repo.lookup(blob_oid).content + oid = repo.write(file_content, :blob) + index = repo.index + index.read_tree(repo.head.target.tree) + index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644) + index.remove("CHANGELOG") + + options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/") + + sha = Rugged::Commit.create(repo, options) + repo.lookup(sha) + end end -- cgit v1.2.1