summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAhmad Sherif <me@ahmadsherif.com>2018-02-28 17:56:00 +0100
committerAhmad Sherif <me@ahmadsherif.com>2018-02-28 21:23:05 +0100
commit7b425c201ef8f7b85f14a08dea4b6125fff7e55f (patch)
treee0a391883f430ffdd6ac458aa6ee7ddd945cecf9
parent0d3ed41ae532812f2529d31a75aec6b103ee0dfa (diff)
downloadgitlab-ce-feature/migrate-recursive-tree-entries-fetching.tar.gz
Fetch commit signatures from Gitaly in batchesfeature/migrate-recursive-tree-entries-fetching
Closes gitaly#1046
-rw-r--r--GITALY_SERVER_VERSION2
-rw-r--r--Gemfile3
-rw-r--r--Gemfile.lock14
-rw-r--r--app/models/commit.rb6
-rw-r--r--lib/gitlab/git/commit.rb43
-rw-r--r--lib/gitlab/gitaly_client/commit_service.rb17
-rw-r--r--lib/gitlab/gpg/commit.rb14
-rw-r--r--spec/lib/gitlab/git/commit_spec.rb130
-rw-r--r--spec/lib/gitlab/gpg/commit_spec.rb12
-rw-r--r--spec/lib/gitlab/gpg/invalid_gpg_signature_updater_spec.rb2
-rw-r--r--spec/services/merge_requests/build_service_spec.rb4
11 files changed, 166 insertions, 81 deletions
diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION
index 137c1281121..6b0ec9bb5bf 100644
--- a/GITALY_SERVER_VERSION
+++ b/GITALY_SERVER_VERSION
@@ -1 +1 @@
-0.85.0
+=feature/implement-get-commits-signature-rpc
diff --git a/Gemfile b/Gemfile
index 6838ddbf01a..06c9533bb28 100644
--- a/Gemfile
+++ b/Gemfile
@@ -411,7 +411,8 @@ group :ed25519 do
end
# Gitaly GRPC client
-gem 'gitaly-proto', '~> 0.85.0', require: 'gitaly'
+# gem 'gitaly-proto', '~> 0.85.0', require: 'gitaly'
+gem 'gitaly-proto', require: 'gitaly', git: 'https://gitlab.com/gitlab-org/gitaly-proto.git', ref: 'add-get-commits-signature-rpc'
# Locked until https://github.com/google/protobuf/issues/4210 is closed
gem 'google-protobuf', '= 3.5.1'
diff --git a/Gemfile.lock b/Gemfile.lock
index 89b86ae0259..80f49669b05 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,3 +1,12 @@
+GIT
+ remote: https://gitlab.com/gitlab-org/gitaly-proto.git
+ revision: be6ba6f75e62f4652500ca97c5c8d8ebd50c671a
+ ref: add-get-commits-signature-rpc
+ specs:
+ gitaly-proto (0.86.0)
+ google-protobuf (~> 3.1)
+ grpc (~> 1.0)
+
GEM
remote: https://rubygems.org/
specs:
@@ -285,9 +294,6 @@ GEM
po_to_json (>= 1.0.0)
rails (>= 3.2.0)
gherkin-ruby (0.3.2)
- gitaly-proto (0.85.0)
- google-protobuf (~> 3.1)
- grpc (~> 1.0)
github-linguist (5.3.3)
charlock_holmes (~> 0.7.5)
escape_utils (~> 1.1.0)
@@ -1057,7 +1063,7 @@ DEPENDENCIES
gettext (~> 3.2.2)
gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.2.0)
- gitaly-proto (~> 0.85.0)
+ gitaly-proto!
github-linguist (~> 5.3.3)
gitlab-flowdock-git-hook (~> 1.0.1)
gitlab-markup (~> 1.6.2)
diff --git a/app/models/commit.rb b/app/models/commit.rb
index add5fcf0e79..b9106309142 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -19,6 +19,7 @@ class Commit
attr_accessor :project, :author
attr_accessor :redacted_description_html
attr_accessor :redacted_title_html
+ attr_reader :gpg_commit
DIFF_SAFE_LINES = Gitlab::Git::DiffCollection::DEFAULT_LIMITS[:max_lines]
@@ -110,6 +111,7 @@ class Commit
@raw = raw_commit
@project = project
@statuses = {}
+ @gpg_commit = Gitlab::Gpg::Commit.new(self) if project
end
def id
@@ -452,8 +454,4 @@ class Commit
def merged_merge_request_no_cache(user)
MergeRequestsFinder.new(user, project_id: project.id).find_by(merge_commit_sha: id) if merge_commit?
end
-
- def gpg_commit
- @gpg_commit ||= Gitlab::Gpg::Commit.new(self)
- end
end
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index ae27a138b7c..e62de51f96d 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -250,6 +250,49 @@ module Gitlab
end
end
+ def extract_signature_lazily(repository, commit_id)
+ BatchLoader.for({ repository: repository, commit_id: commit_id }).batch do |items, loader|
+ items_by_repo = items.group_by { |i| i[:repository] }
+
+ items_by_repo.each do |repo, items|
+ commit_ids = items.map { |i| i[:commit_id] }
+
+ signatures = batch_signature_extraction(repository, commit_ids)
+
+ signatures.each do |commit_id, signature_data|
+ loader.call({ repository: repository, commit_id: commit_id }, signature_data)
+ end
+ end
+ end
+ end
+
+ def batch_signature_extraction(repository, commit_ids)
+ repository.gitaly_migrate(:extract_commit_signature_in_batch) do |is_enabled|
+ if is_enabled
+ gitaly_batch_signature_extraction(repository, commit_ids)
+ else
+ rugged_batch_signature_extraction(repository, commit_ids)
+ end
+ end
+ end
+
+ def gitaly_batch_signature_extraction(repository, commit_ids)
+ repository.gitaly_commit_client.get_commit_signatures(commit_ids)
+ end
+
+ def rugged_batch_signature_extraction(repository, commit_ids)
+ signatures = {}
+
+ commit_ids.each do |commit_id|
+ signature_data = rugged_extract_signature(repository, commit_id)
+ next unless signature_data
+
+ signatures[commit_id] = signature_data
+ end
+
+ signatures
+ end
+
def rugged_extract_signature(repository, commit_id)
begin
Rugged::Commit.extract_signature(repository.rugged, commit_id)
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index d60f57717b5..1ad0bf1d060 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -319,6 +319,23 @@ module Gitlab
[signature, signed_text]
end
+ def get_commit_signatures(commit_ids)
+ request = Gitaly::GetCommitSignaturesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids)
+ response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_signatures, request)
+
+ signatures = Hash.new { |h, k| h[k] = [''.b, ''.b] }
+ current_commit_id = nil
+
+ response.each do |message|
+ current_commit_id = message.commit_id if message.commit_id.present?
+
+ signatures[current_commit_id].first << message.signature
+ signatures[current_commit_id].last << message.signed_text
+ end
+
+ signatures
+ end
+
private
def call_commit_diff(request_params, options = {})
diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb
index 90dd569aaf8..599efaf6c2f 100644
--- a/lib/gitlab/gpg/commit.rb
+++ b/lib/gitlab/gpg/commit.rb
@@ -5,11 +5,19 @@ module Gitlab
@commit = commit
repo = commit.project.repository.raw_repository
- @signature_text, @signed_text = Gitlab::Git::Commit.extract_signature(repo, commit.sha)
+ @signature_data = Gitlab::Git::Commit.extract_signature_lazily(repo, commit.sha || commit.id)
+ end
+
+ def signature_text
+ @signature_data&.itself && @signature_data[0]
+ end
+
+ def signed_text
+ @signature_data&.itself && @signature_data[1]
end
def has_signature?
- !!(@signature_text && @signed_text)
+ !!(signature_text && signed_text)
end
def signature
@@ -53,7 +61,7 @@ module Gitlab
end
def verified_signature
- @verified_signature ||= GPGME::Crypto.new.verify(@signature_text, signed_text: @signed_text) do |verified_signature|
+ @verified_signature ||= GPGME::Crypto.new.verify(signature_text, signed_text: signed_text) do |verified_signature|
break verified_signature
end
end
diff --git a/spec/lib/gitlab/git/commit_spec.rb b/spec/lib/gitlab/git/commit_spec.rb
index 0b20a6349a2..68d224b7b16 100644
--- a/spec/lib/gitlab/git/commit_spec.rb
+++ b/spec/lib/gitlab/git/commit_spec.rb
@@ -393,81 +393,93 @@ describe Gitlab::Git::Commit, seed_helper: true do
end
end
- describe '.extract_signature' do
- subject { described_class.extract_signature(repository, commit_id) }
-
- shared_examples '.extract_signature' do
- context 'when the commit is signed' do
- let(:commit_id) { '0b4bc9a49b562e85de7cc9e834518ea6828729b9' }
-
- it 'returns signature and signed text' do
- signature, signed_text = subject
-
- expected_signature = <<~SIGNATURE
- -----BEGIN PGP SIGNATURE-----
- Version: GnuPG/MacGPG2 v2.0.22 (Darwin)
- Comment: GPGTools - https://gpgtools.org
-
- iQEcBAABCgAGBQJTDvaZAAoJEGJ8X1ifRn8XfvYIAMuB0yrbTGo1BnOSoDfyrjb0
- Kw2EyUzvXYL72B63HMdJ+/0tlSDC6zONF3fc+bBD8z+WjQMTbwFNMRbSSy2rKEh+
- mdRybOP3xBIMGgEph0/kmWln39nmFQBsPRbZBWoU10VfI/ieJdEOgOphszgryRar
- TyS73dLBGE9y9NIININVaNISet9D9QeXFqc761CGjh4YIghvPpi+YihMWapGka6v
- hgKhX+hc5rj+7IEE0CXmlbYR8OYvAbAArc5vJD7UTxAY4Z7/l9d6Ydt9GQ25khfy
- ANFgltYzlR6evLFmDjssiP/mx/ZMN91AL0ueJ9nNGv411Mu2CUW+tDCaQf35mdc=
- =j51i
- -----END PGP SIGNATURE-----
- SIGNATURE
-
- expect(signature).to eq(expected_signature.chomp)
- expect(signature).to be_a_binary_string
-
- expected_signed_text = <<~SIGNED_TEXT
- tree 22bfa2fbd217df24731f43ff43a4a0f8db759dae
- parent ae73cb07c9eeaf35924a10f713b364d32b2dd34f
- author Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> 1393489561 +0200
- committer Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> 1393489561 +0200
-
- Feature added
+ shared_examples 'extracting commit signature' do
+ context 'when the commit is signed' do
+ let(:commit_id) { '0b4bc9a49b562e85de7cc9e834518ea6828729b9' }
+
+ it 'returns signature and signed text' do
+ signature, signed_text = subject
+
+ expected_signature = <<~SIGNATURE
+ -----BEGIN PGP SIGNATURE-----
+ Version: GnuPG/MacGPG2 v2.0.22 (Darwin)
+ Comment: GPGTools - https://gpgtools.org
+
+ iQEcBAABCgAGBQJTDvaZAAoJEGJ8X1ifRn8XfvYIAMuB0yrbTGo1BnOSoDfyrjb0
+ Kw2EyUzvXYL72B63HMdJ+/0tlSDC6zONF3fc+bBD8z+WjQMTbwFNMRbSSy2rKEh+
+ mdRybOP3xBIMGgEph0/kmWln39nmFQBsPRbZBWoU10VfI/ieJdEOgOphszgryRar
+ TyS73dLBGE9y9NIININVaNISet9D9QeXFqc761CGjh4YIghvPpi+YihMWapGka6v
+ hgKhX+hc5rj+7IEE0CXmlbYR8OYvAbAArc5vJD7UTxAY4Z7/l9d6Ydt9GQ25khfy
+ ANFgltYzlR6evLFmDjssiP/mx/ZMN91AL0ueJ9nNGv411Mu2CUW+tDCaQf35mdc=
+ =j51i
+ -----END PGP SIGNATURE-----
+ SIGNATURE
+
+ expect(signature).to eq(expected_signature.chomp)
+ expect(signature).to be_a_binary_string
+
+ expected_signed_text = <<~SIGNED_TEXT
+ tree 22bfa2fbd217df24731f43ff43a4a0f8db759dae
+ parent ae73cb07c9eeaf35924a10f713b364d32b2dd34f
+ author Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> 1393489561 +0200
+ committer Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> 1393489561 +0200
+
+ Feature added
+
+ Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
+ SIGNED_TEXT
+
+ expect(signed_text).to eq(expected_signed_text)
+ expect(signed_text).to be_a_binary_string
+ end
+ end
- Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
- SIGNED_TEXT
+ context 'when the commit has no signature' do
+ let(:commit_id) { '4b4918a572fa86f9771e5ba40fbd48e1eb03e2c6' }
- expect(signed_text).to eq(expected_signed_text)
- expect(signed_text).to be_a_binary_string
- end
+ it 'returns nil' do
+ expect(subject).to be_nil
end
+ end
- context 'when the commit has no signature' do
- let(:commit_id) { '4b4918a572fa86f9771e5ba40fbd48e1eb03e2c6' }
+ context 'when the commit cannot be found' do
+ let(:commit_id) { Gitlab::Git::BLANK_SHA }
- it 'returns nil' do
- expect(subject).to be_nil
- end
+ it 'returns nil' do
+ expect(subject).to be_nil
end
+ end
- context 'when the commit cannot be found' do
- let(:commit_id) { Gitlab::Git::BLANK_SHA }
+ context 'when the commit ID is invalid' do
+ let(:commit_id) { '4b4918a572fa86f9771e5ba40fbd48e' }
- it 'returns nil' do
- expect(subject).to be_nil
- end
+ it 'raises ArgumentError' do
+ expect { subject }.to raise_error(ArgumentError)
end
+ end
+ end
- context 'when the commit ID is invalid' do
- let(:commit_id) { '4b4918a572fa86f9771e5ba40fbd48e' }
+ describe '.extract_signature_lazily' do
+ subject { described_class.extract_signature_lazily(repository, commit_id).itself }
- it 'raises ArgumentError' do
- expect { subject }.to raise_error(ArgumentError)
- end
- end
+ context 'with Gitaly extract_commit_signature_in_batch feature enabled' do
+ it_behaves_like 'extracting commit signature'
+ end
+
+ context 'with Gitaly extract_commit_signature_in_batch feature disabled', :disable_gitaly do
+ it_behaves_like 'extracting commit signature'
end
+ end
+
+ describe '.extract_signature' do
+ subject { described_class.extract_signature(repository, commit_id) }
context 'with gitaly' do
- it_behaves_like '.extract_signature'
+ it_behaves_like 'extracting commit signature'
end
- context 'without gitaly', :skip_gitaly_mock do
- it_behaves_like '.extract_signature'
+ context 'without gitaly', :disable_gitaly do
+ it_behaves_like 'extracting commit signature'
end
end
end
diff --git a/spec/lib/gitlab/gpg/commit_spec.rb b/spec/lib/gitlab/gpg/commit_spec.rb
index 67c62458f0f..8c6d673391b 100644
--- a/spec/lib/gitlab/gpg/commit_spec.rb
+++ b/spec/lib/gitlab/gpg/commit_spec.rb
@@ -38,7 +38,7 @@ describe Gitlab::Gpg::Commit do
end
before do
- allow(Gitlab::Git::Commit).to receive(:extract_signature)
+ allow(Gitlab::Git::Commit).to receive(:extract_signature_lazily)
.with(Gitlab::Git::Repository, commit_sha)
.and_return(
[
@@ -101,7 +101,7 @@ describe Gitlab::Gpg::Commit do
end
before do
- allow(Gitlab::Git::Commit).to receive(:extract_signature)
+ allow(Gitlab::Git::Commit).to receive(:extract_signature_lazily)
.with(Gitlab::Git::Repository, commit_sha)
.and_return(
[
@@ -140,7 +140,7 @@ describe Gitlab::Gpg::Commit do
end
before do
- allow(Gitlab::Git::Commit).to receive(:extract_signature)
+ allow(Gitlab::Git::Commit).to receive(:extract_signature_lazily)
.with(Gitlab::Git::Repository, commit_sha)
.and_return(
[
@@ -175,7 +175,7 @@ describe Gitlab::Gpg::Commit do
end
before do
- allow(Gitlab::Git::Commit).to receive(:extract_signature)
+ allow(Gitlab::Git::Commit).to receive(:extract_signature_lazily)
.with(Gitlab::Git::Repository, commit_sha)
.and_return(
[
@@ -211,7 +211,7 @@ describe Gitlab::Gpg::Commit do
end
before do
- allow(Gitlab::Git::Commit).to receive(:extract_signature)
+ allow(Gitlab::Git::Commit).to receive(:extract_signature_lazily)
.with(Gitlab::Git::Repository, commit_sha)
.and_return(
[
@@ -241,7 +241,7 @@ describe Gitlab::Gpg::Commit do
let!(:commit) { create :commit, project: project, sha: commit_sha }
before do
- allow(Gitlab::Git::Commit).to receive(:extract_signature)
+ allow(Gitlab::Git::Commit).to receive(:extract_signature_lazily)
.with(Gitlab::Git::Repository, commit_sha)
.and_return(
[
diff --git a/spec/lib/gitlab/gpg/invalid_gpg_signature_updater_spec.rb b/spec/lib/gitlab/gpg/invalid_gpg_signature_updater_spec.rb
index c034eccf2a6..6fbffc38444 100644
--- a/spec/lib/gitlab/gpg/invalid_gpg_signature_updater_spec.rb
+++ b/spec/lib/gitlab/gpg/invalid_gpg_signature_updater_spec.rb
@@ -26,7 +26,7 @@ RSpec.describe Gitlab::Gpg::InvalidGpgSignatureUpdater do
before do
allow_any_instance_of(Project).to receive(:commit).and_return(commit)
- allow(Gitlab::Git::Commit).to receive(:extract_signature)
+ allow(Gitlab::Git::Commit).to receive(:extract_signature_lazily)
.with(Gitlab::Git::Repository, commit_sha)
.and_return(signature)
end
diff --git a/spec/services/merge_requests/build_service_spec.rb b/spec/services/merge_requests/build_service_spec.rb
index 3a935d98540..6aed481939e 100644
--- a/spec/services/merge_requests/build_service_spec.rb
+++ b/spec/services/merge_requests/build_service_spec.rb
@@ -15,8 +15,8 @@ describe MergeRequests::BuildService do
let(:target_branch) { 'master' }
let(:merge_request) { service.execute }
let(:compare) { double(:compare, commits: commits) }
- let(:commit_1) { double(:commit_1, safe_message: "Initial commit\n\nCreate the app") }
- let(:commit_2) { double(:commit_2, safe_message: 'This is a bad commit message!') }
+ let(:commit_1) { double(:commit_1, sha: 'f00ba7', safe_message: "Initial commit\n\nCreate the app") }
+ let(:commit_2) { double(:commit_2, sha: 'f00ba7', safe_message: 'This is a bad commit message!') }
let(:commits) { nil }
let(:service) do