GitPushService group but author cross_reference creation18663-commits-reference-mentionables

The reference extractor phase happens once per type not once
pero pushed commit, so we could be avoiding a lot of DB queries

7 files changed, 149 insertions, 29 deletions

diff --git a/lib/banzai/mentionable_renderer.rb b/lib/banzai/mentionable_renderer.rb
new file mode 100644
index 00000000000..c8c37c6910d
--- /dev/null
+++ b/lib/banzai/mentionable_renderer.rb
@@ -0,0 +1,9 @@
+module Banzai
+  module MentionableRenderer
+    def self.render_objects(objects, attr, project, user)
+      renderer = ObjectRenderer.new(project, user)
+
+      renderer.render_objects(objects, attr)
+    end
+  end
+end
diff --git a/lib/banzai/note_renderer.rb b/lib/banzai/note_renderer.rb
index bab6a9934d1..dbcc0d3bf6d 100644
--- a/lib/banzai/note_renderer.rb
+++ b/lib/banzai/note_renderer.rb
@@ -13,8 +13,7 @@ module Banzai
                                     user,
                                     requested_path: path,
                                     project_wiki: wiki,
-                                    ref: git_ref,
-                                    pipeline: :note)
+                                    ref: git_ref)
 
       renderer.render(notes, :note)
     end
diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb
index 9aef807c152..ffd11e2425b 100644
--- a/lib/banzai/object_renderer.rb
+++ b/lib/banzai/object_renderer.rb
@@ -51,9 +51,31 @@ module Banzai
       redactor.redact(documents)
     end
 
+    # Renders the attributes of a set of objects.
+    #
+    # Returns an Array of `Nokogiri::HTML::Document`.
+    def render_attributes(objects, attribute)
+      strings_and_contexts = populate_contexts(objects, attribute)
+
+      Banzai.cache_collection_render(strings_and_contexts).each_with_index.map do |html, index|
+        Banzai::Pipeline[:relative_link].to_document(html, strings_and_contexts[index][:context])
+      end
+    end
+
+    def populate_contexts(objects, attribute)
+      objects.map do |object|
+        context = context_for(object, attribute)
+
+        string = object.__send__(attribute)
+
+        { text: string, context: context }
+      end
+    end
+
     # Returns a Banzai context for the given object and attribute.
     def context_for(object, attribute)
-      context = base_context.merge(cache_key: [object, attribute])
+      context = base_context.merge(base_context_klass_attr(object.class, attribute))
+      context[:cache_key] = [object, attribute]
 
       if object.respond_to?(:author)
         context[:author] = object.author
@@ -62,20 +84,14 @@ module Banzai
       context
     end
 
-    # Renders the attributes of a set of objects.
-    #
-    # Returns an Array of `Nokogiri::HTML::Document`.
-    def render_attributes(objects, attribute)
-      strings_and_contexts = objects.map do |object|
-        context = context_for(object, attribute)
-
-        string = object.__send__(attribute)
+    def base_context_klass_attr(klass, attribute)
+      @base_context_klass_attr ||= Hash.new { |h, k| h[k] = {} }
+      @base_context_klass_attr[klass][attribute] ||= begin
+        return {} unless klass.respond_to?(:mentionable_attrs)
 
-        { text: string, context: context }
-      end
+        _attr, context_klass_options = klass.mentionable_attrs.detect { |attr, _options| attr.to_sym == attribute }
 
-      Banzai.cache_collection_render(strings_and_contexts).each_with_index.map do |html, index|
-        Banzai::Pipeline[:relative_link].to_document(html, strings_and_contexts[index][:context])
+        context_klass_options || {}
       end
     end
 
diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb
index 0df3a72d1c4..edf38e6e067 100644
--- a/lib/banzai/redactor.rb
+++ b/lib/banzai/redactor.rb
@@ -19,7 +19,7 @@ module Banzai
     #
     # Returns the documents passed as the first argument.
     def redact(documents)
-      all_document_nodes = document_nodes(documents)
+      all_document_nodes = ReferenceQuerying.document_nodes(documents)
 
       redact_document_nodes(all_document_nodes)
     end
@@ -28,13 +28,13 @@ module Banzai
     #
     # data - An Array of a Hashes mapping an HTML document to nodes to redact.
     def redact_document_nodes(all_document_nodes)
-      all_nodes = all_document_nodes.map { |x| x[:nodes] }.flatten
+      all_nodes = all_document_nodes.map { |x| x.nodes }.flatten
       visible = nodes_visible_to_user(all_nodes)
       metadata = []
 
       all_document_nodes.each do |entry|
-        nodes_for_document = entry[:nodes]
-        doc_data = { document: entry[:document], visible_reference_count: nodes_for_document.count }
+        nodes_for_document = entry.nodes
+        doc_data = { document: entry.document, visible_reference_count: nodes_for_document.count }
         metadata << doc_data
 
         nodes_for_document.each do |node|
@@ -72,11 +72,5 @@ module Banzai
 
       visible
     end
-
-    def document_nodes(documents)
-      documents.map do |document|
-        { document: document, nodes: Querying.css(document, 'a.gfm[data-reference-type]') }
-      end
-    end
   end
 end
diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb
index f5d110e987b..05f390a9bf8 100644
--- a/lib/banzai/reference_parser/base_parser.rb
+++ b/lib/banzai/reference_parser/base_parser.rb
@@ -36,6 +36,8 @@ module Banzai
         attr_accessor :reference_type
       end
 
+      attr_reader :current_user
+
       # Returns the attribute name containing the value for every object to be
       # parsed by the current parser.
       #
@@ -196,9 +198,9 @@ module Banzai
       end
 
       # Gathers the references for the given HTML nodes.
-      def gather_references(nodes)
-        nodes = nodes_user_can_reference(current_user, nodes)
-        nodes = nodes_visible_to_user(current_user, nodes)
+      def gather_references(nodes, user = self.current_user)
+        nodes = nodes_user_can_reference(user, nodes)
+        nodes = nodes_visible_to_user(user, nodes)
 
         referenced_by(nodes)
       end
@@ -224,7 +226,7 @@ module Banzai
 
       private
 
-      attr_reader :current_user, :project
+      attr_reader :project
 
       def lazy(&block)
         Gitlab::Lazy.new(&block)
diff --git a/lib/banzai/reference_querying.rb b/lib/banzai/reference_querying.rb
new file mode 100644
index 00000000000..37b7f622c36
--- /dev/null
+++ b/lib/banzai/reference_querying.rb
@@ -0,0 +1,37 @@
+module Banzai
+  class ReferenceQuerying
+    def self.document_nodes(documents, types = [])
+      documents.map { |document| DocumentNodes.new(document, types) }
+    end
+
+    class DocumentNodes
+      def initialize(document, types = [])
+        @document = document
+        @types    = types
+      end
+
+      attr_reader :document, :types
+
+      def nodes
+        types.empty? ? raw_nodes : nodes_by_type.values.flatten
+      end
+
+      def nodes_by_type
+        @nodes_by_type ||= begin
+          per_type = Hash.new { |hash, key| hash[key] = [] }
+          raw_nodes.group_by { |node| node.attr('data-reference-type') }.each do |type, nodes|
+            type_sym = type.to_sym
+            per_type[type_sym] = nodes if types.include?(type_sym)
+          end
+          per_type
+        end
+      end
+
+      private
+
+      def raw_nodes
+        @raw_nodes ||= Querying.css(document, 'a.gfm[data-reference-type]')
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cross_reference_extractor.rb b/lib/gitlab/cross_reference_extractor.rb
new file mode 100644
index 00000000000..9dda6f6f298
--- /dev/null
+++ b/lib/gitlab/cross_reference_extractor.rb
@@ -0,0 +1,63 @@
+module Gitlab
+  class CrossReferenceExtractor
+    def initialize(project, user)
+      @project = project
+      @user    = user
+    end
+
+    def references_with_object(objects, attr)
+      documents          = Banzai::MentionableRenderer.render_objects(objects, attr, project, user)
+      all_document_nodes = Banzai::ReferenceQuerying.document_nodes(documents, TYPES)
+      populate_reference_parser_cache(all_document_nodes)
+
+      objects.each_with_index do |object, index|
+        document_nodes = all_document_nodes[index]
+
+        # Here we're going to limit the references from those only on the commit nodes
+        # and that are visible for the commit author.
+        # Using the same parser we ensure the entities are already cached.
+        refs = TYPES.flat_map { |type| redact_references(type, object, document_nodes) }.compact
+
+        yield object, refs
+      end
+    end
+
+    private
+
+    attr_reader :project, :user
+
+    TYPES = %i(issue external_issue merge_request commit)
+
+    def redact_references(type, object, document_nodes)
+      nodes = document_nodes.nodes_by_type[type] || []
+
+      refs = parsers[type].gather_references(nodes, object_author(object)).to_a
+      # From mentionable#referenced_mentionables
+      refs.reject! { |ref| ref == object.local_reference }
+      refs
+    end
+
+    def object_author(object)
+      object.author || user
+    end
+
+    # This method find reference using all the nodes to populate internal reference cache on ther Parser classes
+    # See Banzai::Reference::BaseParser#collection_objects_for_ids
+    def populate_reference_parser_cache(all_document_nodes)
+      TYPES.each do |type|
+        nodes = all_document_nodes.flat_map { |document_nodes| document_nodes.nodes_by_type[type] }
+        parsers[type].referenced_by(nodes).to_a
+      end
+    end
+
+    # We cache parsers because we pretend use their internal caching inverting their execution
+    # order.
+    # 1. we get the references on the whole set of nodes
+    # 2. we get visible_nodes for the user generated nodes
+    def parsers
+      @parsers ||= Hash.new do |hash, type|
+        hash[type] = Banzai::ReferenceParser[type].new(project, user)
+      end
+    end
+  end
+end