From 8044440d7ad8c476d05e3e204ee26b9663738cea Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 14 Aug 2019 21:50:52 -0700 Subject: Eliminate many Gitaly calls in discussions API Previously, the API to retrieve discussions from merge requests often generated hundreds of Gitaly calls to determine whether a system note should be shown to the user. It did this by: 1. Rendering the Markdown 2. Extracting cross-references from the Markdown 3. For cross-references that were commits, a Gitaly FindCommit RPC would be issued to validate that the commit exists. The last step is unnecessary because we don't need to display a commit if the user doesn't have access to the project in the first place. `RendersNotes#prepare_notes_for_rendering` is already used in `MergeRequestsController`, which is why we don't see N+1 Gitaly calls there. We use it here to optimize the note redaction process. Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/65957 --- spec/requests/api/discussions_spec.rb | 53 +++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) (limited to 'spec/requests/api/discussions_spec.rb') diff --git a/spec/requests/api/discussions_spec.rb b/spec/requests/api/discussions_spec.rb index ca1ffe3c524..daf75eb82bb 100644 --- a/spec/requests/api/discussions_spec.rb +++ b/spec/requests/api/discussions_spec.rb @@ -9,6 +9,59 @@ describe API::Discussions do project.add_developer(user) end + context 'with cross-reference system notes', :request_store do + let(:merge_request) { create(:merge_request) } + let(:project) { merge_request.project } + let(:new_merge_request) { create(:merge_request) } + let(:commit) { new_merge_request.project.commit } + let!(:note) { create(:system_note, noteable: merge_request, project: project, note: cross_reference) } + let!(:note_metadata) { create(:system_note_metadata, note: note, action: 'cross_reference') } + let(:cross_reference) { "test commit #{commit.to_reference(project)}" } + let(:url) { "/projects/#{project.id}/merge_requests/#{merge_request.iid}/discussions" } + + before do + project.add_developer(user) + new_merge_request.project.add_developer(user) + end + + it 'returns only the note that the user should see' do + hidden_merge_request = create(:merge_request) + new_cross_reference = "test commit #{hidden_merge_request.project.commit}" + new_note = create(:system_note, noteable: merge_request, project: project, note: new_cross_reference) + create(:system_note_metadata, note: new_note, action: 'cross_reference') + + get api(url, user) + expect(response).to have_gitlab_http_status(200) + expect(json_response.count).to eq(1) + expect(json_response.first['notes'].count).to eq(1) + + parsed_note = json_response.first['notes'].first + expect(parsed_note['id']).to eq(note.id) + expect(parsed_note['body']).to eq(cross_reference) + expect(parsed_note['system']).to be true + end + + it 'avoids Git calls and N+1 SQL queries' do + expect_any_instance_of(Repository).not_to receive(:find_commit).with(commit.id) + + control = ActiveRecord::QueryRecorder.new do + get api(url, user) + end + + expect(response).to have_gitlab_http_status(200) + + RequestStore.clear! + + new_note = create(:system_note, noteable: merge_request, project: project, note: cross_reference) + create(:system_note_metadata, note: new_note, action: 'cross_reference') + + RequestStore.clear! + + expect { get api(url, user) }.not_to exceed_query_limit(control) + expect(response).to have_gitlab_http_status(200) + end + end + context 'when noteable is an Issue' do let!(:issue) { create(:issue, project: project, author: user) } let!(:issue_note) { create(:discussion_note_on_issue, noteable: issue, project: project, author: user) } -- cgit v1.2.1 From b288f128fe0ae20b0db43d207fd87df99dbb551f Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 15 Aug 2019 11:54:42 -0700 Subject: Fix failing N+1 spec in spec/requests/api/discussions_spec.rb This test was failing in EE because the API helper `get()` creates a personal access token each time it's run. We can avoid that by pre-creating a personal access token and passing it each time. --- spec/requests/api/discussions_spec.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'spec/requests/api/discussions_spec.rb') diff --git a/spec/requests/api/discussions_spec.rb b/spec/requests/api/discussions_spec.rb index daf75eb82bb..ef09c6effbb 100644 --- a/spec/requests/api/discussions_spec.rb +++ b/spec/requests/api/discussions_spec.rb @@ -17,6 +17,8 @@ describe API::Discussions do let!(:note) { create(:system_note, noteable: merge_request, project: project, note: cross_reference) } let!(:note_metadata) { create(:system_note_metadata, note: note, action: 'cross_reference') } let(:cross_reference) { "test commit #{commit.to_reference(project)}" } + let(:pat) { create(:personal_access_token, user: user) } + let(:url) { "/projects/#{project.id}/merge_requests/#{merge_request.iid}/discussions" } before do @@ -30,7 +32,7 @@ describe API::Discussions do new_note = create(:system_note, noteable: merge_request, project: project, note: new_cross_reference) create(:system_note_metadata, note: new_note, action: 'cross_reference') - get api(url, user) + get api(url, user, personal_access_token: pat) expect(response).to have_gitlab_http_status(200) expect(json_response.count).to eq(1) expect(json_response.first['notes'].count).to eq(1) @@ -45,7 +47,7 @@ describe API::Discussions do expect_any_instance_of(Repository).not_to receive(:find_commit).with(commit.id) control = ActiveRecord::QueryRecorder.new do - get api(url, user) + get api(url, user, personal_access_token: pat) end expect(response).to have_gitlab_http_status(200) @@ -57,7 +59,7 @@ describe API::Discussions do RequestStore.clear! - expect { get api(url, user) }.not_to exceed_query_limit(control) + expect { get api(url, user, personal_access_token: pat) }.not_to exceed_query_limit(control) expect(response).to have_gitlab_http_status(200) end end -- cgit v1.2.1 From e24b9c2502613cc0df5b2a676236d1c36c02bca4 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 22 Aug 2019 00:19:44 -0700 Subject: Eliminate Gitaly N+1 queries with notes API Similar to https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/31834, we see that in https://gitlab.com/gitlab-org/gitlab-ce/issues/65957 there can be hundreds, even thousands, of Gitaly requests in the `/api/:version/projects/:id/merge_requests/:noteable_id/notes` endpoint. Previously, the API to retrieve notes generated hundreds of Gitaly calls to determine whether a system note should be shown to the user. It did this by: 1. Rendering the Markdown 2. Extracting cross-references from the Markdown 3. Issuing a Gitaly `FindCommit` RPC for every reference to validate that the commit exists. The last step is unnecessary because we don't need to display a commit if the user doesn't have access to the project in the first place. `RendersNotes#prepare_notes_for_rendering` is already used in `MergeRequestsController`, which is why we don't see N+1 Gitaly calls there. We use it here to optimize the note redaction process. --- spec/requests/api/discussions_spec.rb | 54 ++--------------------------------- 1 file changed, 3 insertions(+), 51 deletions(-) (limited to 'spec/requests/api/discussions_spec.rb') diff --git a/spec/requests/api/discussions_spec.rb b/spec/requests/api/discussions_spec.rb index ef09c6effbb..0420201efe3 100644 --- a/spec/requests/api/discussions_spec.rb +++ b/spec/requests/api/discussions_spec.rb @@ -9,59 +9,11 @@ describe API::Discussions do project.add_developer(user) end - context 'with cross-reference system notes', :request_store do - let(:merge_request) { create(:merge_request) } - let(:project) { merge_request.project } - let(:new_merge_request) { create(:merge_request) } - let(:commit) { new_merge_request.project.commit } - let!(:note) { create(:system_note, noteable: merge_request, project: project, note: cross_reference) } - let!(:note_metadata) { create(:system_note_metadata, note: note, action: 'cross_reference') } - let(:cross_reference) { "test commit #{commit.to_reference(project)}" } - let(:pat) { create(:personal_access_token, user: user) } - + context 'when discussions have cross-reference system notes' do let(:url) { "/projects/#{project.id}/merge_requests/#{merge_request.iid}/discussions" } + let(:notes_in_response) { json_response.first['notes'] } - before do - project.add_developer(user) - new_merge_request.project.add_developer(user) - end - - it 'returns only the note that the user should see' do - hidden_merge_request = create(:merge_request) - new_cross_reference = "test commit #{hidden_merge_request.project.commit}" - new_note = create(:system_note, noteable: merge_request, project: project, note: new_cross_reference) - create(:system_note_metadata, note: new_note, action: 'cross_reference') - - get api(url, user, personal_access_token: pat) - expect(response).to have_gitlab_http_status(200) - expect(json_response.count).to eq(1) - expect(json_response.first['notes'].count).to eq(1) - - parsed_note = json_response.first['notes'].first - expect(parsed_note['id']).to eq(note.id) - expect(parsed_note['body']).to eq(cross_reference) - expect(parsed_note['system']).to be true - end - - it 'avoids Git calls and N+1 SQL queries' do - expect_any_instance_of(Repository).not_to receive(:find_commit).with(commit.id) - - control = ActiveRecord::QueryRecorder.new do - get api(url, user, personal_access_token: pat) - end - - expect(response).to have_gitlab_http_status(200) - - RequestStore.clear! - - new_note = create(:system_note, noteable: merge_request, project: project, note: cross_reference) - create(:system_note_metadata, note: new_note, action: 'cross_reference') - - RequestStore.clear! - - expect { get api(url, user, personal_access_token: pat) }.not_to exceed_query_limit(control) - expect(response).to have_gitlab_http_status(200) - end + it_behaves_like 'with cross-reference system notes' end context 'when noteable is an Issue' do -- cgit v1.2.1