From 1b4f767e04388a04372196fc61ba163818aa32e6 Mon Sep 17 00:00:00 2001 From: Ahmad Sherif Date: Fri, 24 Feb 2017 17:53:44 +0200 Subject: Use Gitaly for CommitController#show --- Gemfile | 2 +- Gemfile.lock | 4 +- app/controllers/projects/commit_controller.rb | 9 ++++ app/models/commit.rb | 6 ++- .../feature-use-gitaly-for-commit-show.yml | 4 ++ lib/gitlab/git/diff.rb | 19 +++++++ lib/gitlab/gitaly_client.rb | 14 ++++++ lib/gitlab/gitaly_client/commit.rb | 25 ++++++++++ .../controllers/projects/commit_controller_spec.rb | 10 ++++ spec/lib/gitlab/git/diff_spec.rb | 37 ++++++++++++++ spec/lib/gitlab/gitaly_client/commit_spec.rb | 58 ++++++++++++++++++++++ 11 files changed, 184 insertions(+), 4 deletions(-) create mode 100644 changelogs/unreleased/feature-use-gitaly-for-commit-show.yml create mode 100644 lib/gitlab/gitaly_client/commit.rb create mode 100644 spec/lib/gitlab/gitaly_client/commit_spec.rb diff --git a/Gemfile b/Gemfile index 2f813324a35..6af27ce0f3e 100644 --- a/Gemfile +++ b/Gemfile @@ -352,4 +352,4 @@ gem 'vmstat', '~> 2.3.0' gem 'sys-filesystem', '~> 1.1.6' # Gitaly GRPC client -gem 'gitaly', '~> 0.2.1' +gem 'gitaly', '~> 0.3.0' diff --git a/Gemfile.lock b/Gemfile.lock index 62388628eaa..9e649dbbe3a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -250,7 +250,7 @@ GEM json get_process_mem (0.2.0) gherkin-ruby (0.3.2) - gitaly (0.2.1) + gitaly (0.3.0) google-protobuf (~> 3.1) grpc (~> 1.0) github-linguist (4.7.6) @@ -896,7 +896,7 @@ DEPENDENCIES fuubar (~> 2.0.0) gemnasium-gitlab-service (~> 0.2) gemojione (~> 3.0) - gitaly (~> 0.2.1) + gitaly (~> 0.3.0) github-linguist (~> 4.7.0) gitlab-flowdock-git-hook (~> 1.0.1) gitlab-markup (~> 1.5.1) diff --git a/app/controllers/projects/commit_controller.rb b/app/controllers/projects/commit_controller.rb index cc67f688d51..b69d80459bf 100644 --- a/app/controllers/projects/commit_controller.rb +++ b/app/controllers/projects/commit_controller.rb @@ -6,6 +6,7 @@ class Projects::CommitController < Projects::ApplicationController include DiffForPath include DiffHelper + around_action :migrate_to_gitaly, only: [:show] # Authorize before_action :require_non_empty_project before_action :authorize_download_code! @@ -103,6 +104,7 @@ class Projects::CommitController < Projects::ApplicationController opts = diff_options opts[:ignore_whitespace_change] = true if params[:format] == 'diff' + opts[:use_gitaly] = @use_gitaly @diffs = commit.diffs(opts) @notes_count = commit.notes.count @@ -133,4 +135,11 @@ class Projects::CommitController < Projects::ApplicationController @start_branch = params[:start_branch] @commit_params = { commit: @commit } end + + def migrate_to_gitaly + Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| + @use_gitaly = is_enabled + yield + end + end end diff --git a/app/models/commit.rb b/app/models/commit.rb index 0a18986ef26..5c4fe3ea77c 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -317,7 +317,11 @@ class Commit end def raw_diffs(*args) - raw.diffs(*args) + if args.last.is_a?(Hash) && args.last[:use_gitaly] + Gitlab::GitalyClient::Commit.diff_from_parent(self, *args) + else + raw.diffs(*args) + end end def diffs(diff_options = nil) diff --git a/changelogs/unreleased/feature-use-gitaly-for-commit-show.yml b/changelogs/unreleased/feature-use-gitaly-for-commit-show.yml new file mode 100644 index 00000000000..4b668d994a1 --- /dev/null +++ b/changelogs/unreleased/feature-use-gitaly-for-commit-show.yml @@ -0,0 +1,4 @@ +--- +title: Use Gitaly for CommitController#show +merge_request: 9629 +author: diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 2a017c93f57..276e8e595a0 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -19,6 +19,9 @@ module Gitlab # The maximum size before a diff is collapsed. DIFF_COLLAPSE_LIMIT = 10240 # 10 KB + # Blank SHA + BLANK_SHA = ("0" * 40).freeze + class << self def between(repo, head, base, options = {}, *paths) straight = options.delete(:straight) || false @@ -179,6 +182,8 @@ module Gitlab init_from_hash(raw_diff, collapse: collapse) when Rugged::Patch, Rugged::Diff::Delta init_from_rugged(raw_diff, collapse: collapse) + when Gitaly::CommitDiffResponse + init_from_gitaly(raw_diff, collapse: collapse) when nil raise "Nil as raw diff passed" else @@ -277,6 +282,20 @@ module Gitlab prune_collapsed_diff! if collapse && collapsible? end + def init_from_gitaly(diff_msg, collapse: false) + @diff = diff_msg.raw_chunks.join + @new_path = encode!(diff_msg.to_path.dup) + @old_path = encode!(diff_msg.from_path.dup) + @a_mode = diff_msg.old_mode.to_s(8) + @b_mode = diff_msg.new_mode.to_s(8) + @new_file = diff_msg.from_id == BLANK_SHA + @renamed_file = diff_msg.from_path != diff_msg.to_path + @deleted_file = diff_msg.to_id == BLANK_SHA + + prune_large_diff! if too_large? + prune_collapsed_diff! if collapse && collapsible? + end + # If the patch surpasses any of the diff limits it calls the appropiate # prune method and returns true. Otherwise returns false. def prune_large_patch(patch, collapse) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index b981a629fb0..5534d4af439 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -25,5 +25,19 @@ module Gitlab def self.enabled? gitaly_address.present? end + + def self.feature_enabled?(feature) + enabled? && ENV["GITALY_#{feature.upcase}"] == '1' + end + + def self.migrate(feature) + is_enabled = feature_enabled?(feature) + metric_name = feature.to_s + metric_name += "_gitaly" if is_enabled + + Gitlab::Metrics.measure(metric_name) do + yield is_enabled + end + end end end diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb new file mode 100644 index 00000000000..525b8d680e9 --- /dev/null +++ b/lib/gitlab/gitaly_client/commit.rb @@ -0,0 +1,25 @@ +module Gitlab + module GitalyClient + class Commit + # The ID of empty tree. + # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 + EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze + + class << self + def diff_from_parent(commit, options = {}) + stub = Gitaly::Diff::Stub.new(nil, nil, channel_override: GitalyClient.channel) + repo = Gitaly::Repository.new(path: commit.project.repository.path_to_repo) + parent = commit.parents[0] + parent_id = parent ? parent.id : EMPTY_TREE_ID + request = Gitaly::CommitDiffRequest.new( + repository: repo, + left_commit_id: parent_id, + right_commit_id: commit.id + ) + + Gitlab::Git::DiffCollection.new(stub.commit_diff(request), options) + end + end + end + end +end diff --git a/spec/controllers/projects/commit_controller_spec.rb b/spec/controllers/projects/commit_controller_spec.rb index b223a22ae60..3da4c6bc87e 100644 --- a/spec/controllers/projects/commit_controller_spec.rb +++ b/spec/controllers/projects/commit_controller_spec.rb @@ -132,6 +132,16 @@ describe Projects::CommitController do expect(response).to be_success end end + + context 'Gitaly commit_raw_diffs feature enabled' do + it 'fetches diff from Gitaly server' do + allow(Gitlab::GitalyClient).to receive(:feature_enabled?).with(:commit_raw_diffs).and_return(true) + expect(Gitlab::GitalyClient::Commit).to receive(:diff_from_parent). + with(commit, instance_of(Hash)).and_return(Gitlab::Git::DiffCollection.new([])) + + go(id: commit.id) + end + end end describe "GET branches" do diff --git a/spec/lib/gitlab/git/diff_spec.rb b/spec/lib/gitlab/git/diff_spec.rb index 4c55532d165..992126ef153 100644 --- a/spec/lib/gitlab/git/diff_spec.rb +++ b/spec/lib/gitlab/git/diff_spec.rb @@ -109,6 +109,43 @@ EOT end end end + + context 'using a Gitaly::CommitDiffResponse' do + let(:diff) do + described_class.new( + Gitaly::CommitDiffResponse.new( + to_path: ".gitmodules", + from_path: ".gitmodules", + old_mode: 0100644, + new_mode: 0100644, + from_id: '357406f3075a57708d0163752905cc1576fceacc', + to_id: '8e5177d718c561d36efde08bad36b43687ee6bf0', + raw_chunks: raw_chunks, + ) + ) + end + + context 'with a small diff' do + let(:raw_chunks) { [@raw_diff_hash[:diff]] } + + it 'initializes the diff' do + expect(diff.to_hash).to eq(@raw_diff_hash) + end + + it 'does not prune the diff' do + expect(diff).not_to be_too_large + end + end + + context 'using a diff that is too large' do + let(:raw_chunks) { ['a' * 204800] } + + it 'prunes the diff' do + expect(diff.diff).to be_empty + expect(diff).to be_too_large + end + end + end end describe 'straight diffs' do diff --git a/spec/lib/gitlab/gitaly_client/commit_spec.rb b/spec/lib/gitlab/gitaly_client/commit_spec.rb new file mode 100644 index 00000000000..4d812b876e4 --- /dev/null +++ b/spec/lib/gitlab/gitaly_client/commit_spec.rb @@ -0,0 +1,58 @@ +require 'spec_helper' + +describe Gitlab::GitalyClient::Commit do + describe '.diff_from_parent' do + let(:diff_stub) { double('Gitaly::Diff::Stub') } + let(:project) { create(:project) } + let(:repository_message) { Gitaly::Repository.new(path: project.repository.path) } + let(:commit) { project.commit('913c66a37b4a45b9769037c55c2d238bd0942d2e') } + + before do + allow(Gitaly::Diff::Stub).to receive(:new).and_return(diff_stub) + allow(diff_stub).to receive(:commit_diff).and_return([]) + end + + context 'when a commit has a parent' do + it 'sends an RPC request with the parent ID as left commit' do + request = Gitaly::CommitDiffRequest.new( + repository: repository_message, + left_commit_id: 'cfe32cf61b73a0d5e9f13e774abde7ff789b1660', + right_commit_id: commit.id, + ) + + expect(diff_stub).to receive(:commit_diff).with(request) + + described_class.diff_from_parent(commit) + end + end + + context 'when a commit does not have a parent' do + it 'sends an RPC request with empty tree ref as left commit' do + initial_commit = project.commit('1a0b36b3cdad1d2ee32457c102a8c0b7056fa863') + request = Gitaly::CommitDiffRequest.new( + repository: repository_message, + left_commit_id: '4b825dc642cb6eb9a060e54bf8d69288fbee4904', + right_commit_id: initial_commit.id, + ) + + expect(diff_stub).to receive(:commit_diff).with(request) + + described_class.diff_from_parent(initial_commit) + end + end + + it 'returns a Gitlab::Git::DiffCollection' do + ret = described_class.diff_from_parent(commit) + + expect(ret).to be_kind_of(Gitlab::Git::DiffCollection) + end + + it 'passes options to Gitlab::Git::DiffCollection' do + options = { max_files: 31, max_lines: 13 } + + expect(Gitlab::Git::DiffCollection).to receive(:new).with([], options) + + described_class.diff_from_parent(commit, options) + end + end +end -- cgit v1.2.1