Eliminate Gitaly N+1 queries with notes APIsh-eliminate-gitaly-nplus-one-notes

Similar to https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/31834,
we see that in https://gitlab.com/gitlab-org/gitlab-ce/issues/65957
there can be hundreds, even thousands, of Gitaly requests in the
`/api/:version/projects/:id/merge_requests/:noteable_id/notes` endpoint.

Previously, the API to retrieve notes generated hundreds of Gitaly calls
to determine whether a system note should be shown to the user. It did
this by:

1. Rendering the Markdown
2. Extracting cross-references from the Markdown
3. Issuing a Gitaly `FindCommit` RPC for every reference to validate
that the commit exists.

The last step is unnecessary because we don't need to display a commit
if the user doesn't have access to the project in the first place.
`RendersNotes#prepare_notes_for_rendering` is already used in
`MergeRequestsController`, which is why we don't see N+1 Gitaly calls
there. We use it here to optimize the note redaction process.

6 files changed, 78 insertions, 57 deletions

diff --git a/changelogs/unreleased/sh-eliminate-gitaly-nplus-one-notes.yml b/changelogs/unreleased/sh-eliminate-gitaly-nplus-one-notes.yml
new file mode 100644
index 00000000000..00523f53dd7
--- /dev/null
+++ b/changelogs/unreleased/sh-eliminate-gitaly-nplus-one-notes.yml
@@ -0,0 +1,5 @@
+---
+title: Eliminate Gitaly N+1 queries with notes API
+merge_request: 32089
+author:
+type: performance
diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb
index 6bf9057fad7..b2bf6bf7417 100644
--- a/lib/api/helpers/notes_helpers.rb
+++ b/lib/api/helpers/notes_helpers.rb
@@ -3,6 +3,8 @@
 module API
   module Helpers
     module NotesHelpers
+      include ::RendersNotes
+
       def self.noteable_types
         # This is a method instead of a constant, allowing EE to more easily
         # extend it.
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index 9381f045144..84563d66ee8 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -36,12 +36,13 @@ module API
           # page can have less elements than :per_page even if
           # there's more than one page.
           raw_notes = noteable.notes.with_metadata.reorder(order_options_with_tie_breaker)
-          notes =
-            # paginate() only works with a relation. This could lead to a
-            # mismatch between the pagination headers info and the actual notes
-            # array returned, but this is really a edge-case.
-            paginate(raw_notes)
-            .reject { |n| n.cross_reference_not_visible_for?(current_user) }
+
+          # paginate() only works with a relation. This could lead to a
+          # mismatch between the pagination headers info and the actual notes
+          # array returned, but this is really a edge-case.
+          notes = paginate(raw_notes)
+          notes = prepare_notes_for_rendering(notes)
+          notes = notes.reject { |n| n.cross_reference_not_visible_for?(current_user) }
           present notes, with: Entities::Note
         end
         # rubocop: enable CodeReuse/ActiveRecord
diff --git a/spec/requests/api/discussions_spec.rb b/spec/requests/api/discussions_spec.rb
index ef09c6effbb..0420201efe3 100644
--- a/spec/requests/api/discussions_spec.rb
+++ b/spec/requests/api/discussions_spec.rb
@@ -9,59 +9,11 @@ describe API::Discussions do
     project.add_developer(user)
   end
 
-  context 'with cross-reference system notes', :request_store do
-    let(:merge_request) { create(:merge_request) }
-    let(:project) { merge_request.project }
-    let(:new_merge_request) { create(:merge_request) }
-    let(:commit) { new_merge_request.project.commit }
-    let!(:note) { create(:system_note, noteable: merge_request, project: project, note: cross_reference) }
-    let!(:note_metadata) { create(:system_note_metadata, note: note, action: 'cross_reference') }
-    let(:cross_reference) { "test commit #{commit.to_reference(project)}" }
-    let(:pat) { create(:personal_access_token, user: user) }
-
+  context 'when discussions have cross-reference system notes' do
     let(:url) { "/projects/#{project.id}/merge_requests/#{merge_request.iid}/discussions" }
+    let(:notes_in_response) { json_response.first['notes'] }
 
-    before do
-      project.add_developer(user)
-      new_merge_request.project.add_developer(user)
-    end
-
-    it 'returns only the note that the user should see' do
-      hidden_merge_request = create(:merge_request)
-      new_cross_reference = "test commit #{hidden_merge_request.project.commit}"
-      new_note = create(:system_note, noteable: merge_request, project: project, note: new_cross_reference)
-      create(:system_note_metadata, note: new_note, action: 'cross_reference')
-
-      get api(url, user, personal_access_token: pat)
-      expect(response).to have_gitlab_http_status(200)
-      expect(json_response.count).to eq(1)
-      expect(json_response.first['notes'].count).to eq(1)
-
-      parsed_note = json_response.first['notes'].first
-      expect(parsed_note['id']).to eq(note.id)
-      expect(parsed_note['body']).to eq(cross_reference)
-      expect(parsed_note['system']).to be true
-    end
-
-    it 'avoids Git calls and N+1 SQL queries' do
-      expect_any_instance_of(Repository).not_to receive(:find_commit).with(commit.id)
-
-      control = ActiveRecord::QueryRecorder.new do
-        get api(url, user, personal_access_token: pat)
-      end
-
-      expect(response).to have_gitlab_http_status(200)
-
-      RequestStore.clear!
-
-      new_note = create(:system_note, noteable: merge_request, project: project, note: cross_reference)
-      create(:system_note_metadata, note: new_note, action: 'cross_reference')
-
-      RequestStore.clear!
-
-      expect { get api(url, user, personal_access_token: pat) }.not_to exceed_query_limit(control)
-      expect(response).to have_gitlab_http_status(200)
-    end
+    it_behaves_like 'with cross-reference system notes'
   end
 
   context 'when noteable is an Issue' do
diff --git a/spec/requests/api/notes_spec.rb b/spec/requests/api/notes_spec.rb
index 424f0a82e43..6c1e30791d2 100644
--- a/spec/requests/api/notes_spec.rb
+++ b/spec/requests/api/notes_spec.rb
@@ -9,6 +9,13 @@ describe API::Notes do
     project.add_reporter(user)
   end
 
+  context 'when there are cross-reference system notes' do
+    let(:url) { "/projects/#{project.id}/merge_requests/#{merge_request.iid}/notes" }
+    let(:notes_in_response) { json_response }
+
+    it_behaves_like 'with cross-reference system notes'
+  end
+
   context "when noteable is an Issue" do
     let!(:issue) { create(:issue, project: project, author: user) }
     let!(:issue_note) { create(:note, noteable: issue, project: project, author: user) }
diff --git a/spec/support/shared_examples/requests/api/discussions.rb b/spec/support/shared_examples/requests/api/discussions.rb
index fc72287f265..a36bc2dc9b5 100644
--- a/spec/support/shared_examples/requests/api/discussions.rb
+++ b/spec/support/shared_examples/requests/api/discussions.rb
@@ -1,5 +1,59 @@
 # frozen_string_literal: true
 
+shared_examples 'with cross-reference system notes' do
+  let(:merge_request) { create(:merge_request) }
+  let(:project) { merge_request.project }
+  let(:new_merge_request) { create(:merge_request) }
+  let(:commit) { new_merge_request.project.commit }
+  let!(:note) { create(:system_note, noteable: merge_request, project: project, note: cross_reference) }
+  let!(:note_metadata) { create(:system_note_metadata, note: note, action: 'cross_reference') }
+  let(:cross_reference) { "test commit #{commit.to_reference(project)}" }
+  let(:pat) { create(:personal_access_token, user: user) }
+
+  before do
+    project.add_developer(user)
+    new_merge_request.project.add_developer(user)
+
+    hidden_merge_request = create(:merge_request)
+    new_cross_reference = "test commit #{hidden_merge_request.project.commit}"
+    new_note = create(:system_note, noteable: merge_request, project: project, note: new_cross_reference)
+    create(:system_note_metadata, note: new_note, action: 'cross_reference')
+  end
+
+  it 'returns only the note that the user should see' do
+    get api(url, user, personal_access_token: pat)
+
+    expect(response).to have_gitlab_http_status(200)
+    expect(json_response.count).to eq(1)
+    expect(notes_in_response.count).to eq(1)
+
+    parsed_note = notes_in_response.first
+    expect(parsed_note['id']).to eq(note.id)
+    expect(parsed_note['body']).to eq(cross_reference)
+    expect(parsed_note['system']).to be true
+  end
+
+  it 'avoids Git calls and N+1 SQL queries', :request_store do
+    expect_any_instance_of(Repository).not_to receive(:find_commit).with(commit.id)
+
+    control = ActiveRecord::QueryRecorder.new do
+      get api(url, user, personal_access_token: pat)
+    end
+
+    expect(response).to have_gitlab_http_status(200)
+
+    RequestStore.clear!
+
+    new_note = create(:system_note, noteable: merge_request, project: project, note: cross_reference)
+    create(:system_note_metadata, note: new_note, action: 'cross_reference')
+
+    RequestStore.clear!
+
+    expect { get api(url, user, personal_access_token: pat) }.not_to exceed_query_limit(control)
+    expect(response).to have_gitlab_http_status(200)
+  end
+end
+
 shared_examples 'discussions API' do |parent_type, noteable_type, id_name, can_reply_to_individual_notes: false|
   describe "GET /#{parent_type}/:id/#{noteable_type}/:noteable_id/discussions" do
     it "returns an array of discussions" do