diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-05-19 15:44:42 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-05-19 15:44:42 +0000 |
commit | 4555e1b21c365ed8303ffb7a3325d773c9b8bf31 (patch) | |
tree | 5423a1c7516cffe36384133ade12572cf709398d /lib/banzai | |
parent | e570267f2f6b326480d284e0164a6464ba4081bc (diff) | |
download | gitlab-ce-4555e1b21c365ed8303ffb7a3325d773c9b8bf31.tar.gz |
Add latest changes from gitlab-org/gitlab@13-12-stable-eev13.12.0-rc42
Diffstat (limited to 'lib/banzai')
36 files changed, 433 insertions, 404 deletions
diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index b7344808989..1bf6cf11526 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -17,7 +17,12 @@ module Banzai return context[:project] || context[:group] unless ref return context[:project] if context[:project]&.full_path == ref - Project.find_by_full_path(ref) + if reference_cache.cache_loaded? + # optimization to reuse the parent_per_reference query information + reference_cache.parent_per_reference[ref || reference_cache.current_parent_path] + else + Project.find_by_full_path(ref) + end end end end diff --git a/lib/banzai/filter/base_relative_link_filter.rb b/lib/banzai/filter/base_relative_link_filter.rb index fd526df4c48..84a6e18e77b 100644 --- a/lib/banzai/filter/base_relative_link_filter.rb +++ b/lib/banzai/filter/base_relative_link_filter.rb @@ -10,19 +10,16 @@ module Banzai protected def linkable_attributes - strong_memoize(:linkable_attributes) do - attrs = [] - - attrs += doc.search('a:not(.gfm)').map do |el| - el.attribute('href') - end - - attrs += doc.search('img:not(.gfm), video:not(.gfm), audio:not(.gfm)').flat_map do |el| - [el.attribute('src'), el.attribute('data-src')] - end - - attrs.reject do |attr| - attr.blank? || attr.value.start_with?('//') + if Feature.enabled?(:optimize_linkable_attributes, project, default_enabled: :yaml) + # Nokorigi Nodeset#search performs badly for documents with many nodes + # + # Here we store fetched attributes in the shared variable "result" + # This variable is passed through the chain of filters and can be + # accessed by them + result[:linkable_attributes] ||= fetch_linkable_attributes + else + strong_memoize(:linkable_attributes) do + fetch_linkable_attributes end end end @@ -40,6 +37,16 @@ module Banzai def unescape_and_scrub_uri(uri) Addressable::URI.unescape(uri).scrub.delete("\0") end + + def fetch_linkable_attributes + attrs = [] + + attrs += doc.search('a:not(.gfm), img:not(.gfm), video:not(.gfm), audio:not(.gfm)').flat_map do |el| + [el.attribute('href'), el.attribute('src'), el.attribute('data-src')] + end + + attrs.reject { |attr| attr.blank? || attr.value.start_with?('//') } + end end end end diff --git a/lib/banzai/filter/custom_emoji_filter.rb b/lib/banzai/filter/custom_emoji_filter.rb index 1ee8f4e31e8..3171231dc9b 100644 --- a/lib/banzai/filter/custom_emoji_filter.rb +++ b/lib/banzai/filter/custom_emoji_filter.rb @@ -3,6 +3,8 @@ module Banzai module Filter class CustomEmojiFilter < HTML::Pipeline::Filter + include Gitlab::Utils::StrongMemoize + IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set def call @@ -14,7 +16,7 @@ module Banzai next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) next unless content.include?(':') - next unless namespace && namespace.custom_emoji.any? + next unless has_custom_emoji? html = custom_emoji_name_element_filter(content) @@ -46,6 +48,12 @@ module Banzai private + def has_custom_emoji? + strong_memoize(:has_custom_emoji) do + namespace&.custom_emoji&.any? + end + end + def namespace context[:project].namespace.root_ancestor end diff --git a/lib/banzai/filter/markdown_pre_escape_filter.rb b/lib/banzai/filter/markdown_pre_escape_filter.rb index bedc2d0fd04..0c53444681d 100644 --- a/lib/banzai/filter/markdown_pre_escape_filter.rb +++ b/lib/banzai/filter/markdown_pre_escape_filter.rb @@ -26,7 +26,7 @@ module Banzai class MarkdownPreEscapeFilter < HTML::Pipeline::TextFilter # We just need to target those that are special GitLab references REFERENCE_CHARACTERS = '@#!$&~%^' - ASCII_PUNCTUATION = %r{([\\][#{REFERENCE_CHARACTERS}])}.freeze + ASCII_PUNCTUATION = %r{(\\[#{REFERENCE_CHARACTERS}])}.freeze LITERAL_KEYWORD = 'cmliteral' def call diff --git a/lib/banzai/filter/references/abstract_reference_filter.rb b/lib/banzai/filter/references/abstract_reference_filter.rb index 7109373dbce..08014ccdcce 100644 --- a/lib/banzai/filter/references/abstract_reference_filter.rb +++ b/lib/banzai/filter/references/abstract_reference_filter.rb @@ -8,6 +8,12 @@ module Banzai class AbstractReferenceFilter < ReferenceFilter include CrossProjectReference + def initialize(doc, context = nil, result = nil) + super + + @reference_cache = ReferenceCache.new(self, context) + end + # REFERENCE_PLACEHOLDER is used for re-escaping HTML text except found # reference (which we replace with placeholder during re-scaping). The # random number helps ensure it's pretty close to unique. Since it's a @@ -16,22 +22,9 @@ module Banzai REFERENCE_PLACEHOLDER = "_reference_#{SecureRandom.hex(16)}_" REFERENCE_PLACEHOLDER_PATTERN = %r{#{REFERENCE_PLACEHOLDER}(\d+)}.freeze - def self.object_class - # Implement in child class - # Example: MergeRequest - end - - def self.object_name - @object_name ||= object_class.name.underscore - end - - def self.object_sym - @object_sym ||= object_name.to_sym - end - # Public: Find references in text (like `!123` for merge requests) # - # AnyReferenceFilter.references_in(text) do |match, id, project_ref, matches| + # references_in(text) do |match, id, project_ref, matches| # object = find_object(project_ref, id) # "<a href=...>#{object.to_reference}</a>" # end @@ -42,7 +35,7 @@ module Banzai # of the external project reference, and all of the matchdata. # # Returns a String replaced with the return of the block. - def self.references_in(text, pattern = object_class.reference_pattern) + def references_in(text, pattern = object_class.reference_pattern) text.gsub(pattern) do |match| if ident = identifier($~) yield match, ident, $~[:project], $~[:namespace], $~ @@ -52,17 +45,13 @@ module Banzai end end - def self.identifier(match_data) + def identifier(match_data) symbol = symbol_from_match(match_data) parse_symbol(symbol, match_data) if object_class.reference_valid?(symbol) end - def identifier(match_data) - self.class.identifier(match_data) - end - - def self.symbol_from_match(match) + def symbol_from_match(match) key = object_sym match[key] if match.names.include?(key.to_s) end @@ -72,7 +61,7 @@ module Banzai # # This method has the contract that if a string `ref` refers to a # record `record`, then `parse_symbol(ref) == record_identifier(record)`. - def self.parse_symbol(symbol, match_data) + def parse_symbol(symbol, match_data) symbol.to_i end @@ -84,21 +73,10 @@ module Banzai record.id end - def object_class - self.class.object_class - end - - def object_sym - self.class.object_sym - end - - def references_in(*args, &block) - self.class.references_in(*args, &block) - end - # Implement in child class # Example: project.merge_requests.find def find_object(parent_object, id) + raise NotImplementedError, "#{self.class} must implement method: #{__callee__}" end # Override if the link reference pattern produces a different ID (global @@ -110,6 +88,7 @@ module Banzai # Implement in child class # Example: project_merge_request_url def url_for_object(object, parent_object) + raise NotImplementedError, "#{self.class} must implement method: #{__callee__}" end def find_object_cached(parent_object, id) @@ -139,7 +118,9 @@ module Banzai def call return doc unless project || group || user - ref_pattern = object_class.reference_pattern + reference_cache.load_reference_cache(nodes) if respond_to?(:parent_records) + + ref_pattern = object_reference_pattern link_pattern = object_class.link_reference_pattern # Compile often used regexps only once outside of the loop @@ -201,9 +182,9 @@ module Banzai def object_link_filter(text, pattern, link_content: nil, link_reference: false) references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches| parent_path = if parent_type == :group - full_group_path(namespace_ref) + reference_cache.full_group_path(namespace_ref) else - full_project_path(namespace_ref, project_ref) + reference_cache.full_project_path(namespace_ref, project_ref) end parent = from_ref_cached(parent_path) @@ -290,127 +271,6 @@ module Banzai text end - # Returns a Hash containing all object references (e.g. issue IDs) per the - # project they belong to. - def references_per_parent - @references_per ||= {} - - @references_per[parent_type] ||= begin - refs = Hash.new { |hash, key| hash[key] = Set.new } - regex = [ - object_class.link_reference_pattern, - object_class.reference_pattern - ].compact.reduce { |a, b| Regexp.union(a, b) } - - nodes.each do |node| - node.to_html.scan(regex) do - path = if parent_type == :project - full_project_path($~[:namespace], $~[:project]) - else - full_group_path($~[:group]) - end - - if ident = identifier($~) - refs[path] << ident - end - end - end - - refs - end - end - - # Returns a Hash containing referenced projects grouped per their full - # path. - def parent_per_reference - @per_reference ||= {} - - @per_reference[parent_type] ||= begin - refs = Set.new - - references_per_parent.each do |ref, _| - refs << ref - end - - find_for_paths(refs.to_a).index_by(&:full_path) - end - end - - def relation_for_paths(paths) - klass = parent_type.to_s.camelize.constantize - result = klass.where_full_path_in(paths) - return result if parent_type == :group - - result.includes(:namespace) if parent_type == :project - end - - # Returns projects for the given paths. - def find_for_paths(paths) - if Gitlab::SafeRequestStore.active? - cache = refs_cache - to_query = paths - cache.keys - - unless to_query.empty? - records = relation_for_paths(to_query) - - found = [] - records.each do |record| - ref = record.full_path - get_or_set_cache(cache, ref) { record } - found << ref - end - - not_found = to_query - found - not_found.each do |ref| - get_or_set_cache(cache, ref) { nil } - end - end - - cache.slice(*paths).values.compact - else - relation_for_paths(paths) - end - end - - def current_parent_path - @current_parent_path ||= parent&.full_path - end - - def current_project_namespace_path - @current_project_namespace_path ||= project&.namespace&.full_path - end - - def records_per_parent - @_records_per_project ||= {} - - @_records_per_project[object_class.to_s.underscore] ||= begin - hash = Hash.new { |h, k| h[k] = {} } - - parent_per_reference.each do |path, parent| - record_ids = references_per_parent[path] - - parent_records(parent, record_ids).each do |record| - hash[parent][record_identifier(record)] = record - end - end - - hash - end - end - - private - - def full_project_path(namespace, project_ref) - return current_parent_path unless project_ref - - namespace_ref = namespace || current_project_namespace_path - "#{namespace_ref}/#{project_ref}" - end - - def refs_cache - Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} - end - def parent_type :project end @@ -419,19 +279,9 @@ module Banzai parent_type == :project ? project : group end - def full_group_path(group_ref) - return current_parent_path unless group_ref - - group_ref - end - - def unescape_html_entities(text) - CGI.unescapeHTML(text.to_s) - end + private - def escape_html_entities(text) - CGI.escapeHTML(text.to_s) - end + attr_accessor :reference_cache def escape_with_placeholders(text, placeholder_data) escaped = escape_html_entities(text) @@ -444,5 +294,3 @@ module Banzai end end end - -Banzai::Filter::References::AbstractReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::AbstractReferenceFilter') diff --git a/lib/banzai/filter/references/alert_reference_filter.rb b/lib/banzai/filter/references/alert_reference_filter.rb index 90fef536605..512d4028520 100644 --- a/lib/banzai/filter/references/alert_reference_filter.rb +++ b/lib/banzai/filter/references/alert_reference_filter.rb @@ -5,12 +5,9 @@ module Banzai module References class AlertReferenceFilter < IssuableReferenceFilter self.reference_type = :alert + self.object_class = AlertManagement::Alert - def self.object_class - AlertManagement::Alert - end - - def self.object_sym + def object_sym :alert end diff --git a/lib/banzai/filter/references/commit_range_reference_filter.rb b/lib/banzai/filter/references/commit_range_reference_filter.rb index ad79f8a173c..df7f42eaa70 100644 --- a/lib/banzai/filter/references/commit_range_reference_filter.rb +++ b/lib/banzai/filter/references/commit_range_reference_filter.rb @@ -8,12 +8,9 @@ module Banzai # This filter supports cross-project references. class CommitRangeReferenceFilter < AbstractReferenceFilter self.reference_type = :commit_range + self.object_class = CommitRange - def self.object_class - CommitRange - end - - def self.references_in(text, pattern = CommitRange.reference_pattern) + def references_in(text, pattern = object_reference_pattern) text.gsub(pattern) do |match| yield match, $~[:commit_range], $~[:project], $~[:namespace], $~ end diff --git a/lib/banzai/filter/references/commit_reference_filter.rb b/lib/banzai/filter/references/commit_reference_filter.rb index 457921bd07d..157dc696cc8 100644 --- a/lib/banzai/filter/references/commit_reference_filter.rb +++ b/lib/banzai/filter/references/commit_reference_filter.rb @@ -8,12 +8,9 @@ module Banzai # This filter supports cross-project references. class CommitReferenceFilter < AbstractReferenceFilter self.reference_type = :commit + self.object_class = Commit - def self.object_class - Commit - end - - def self.references_in(text, pattern = Commit.reference_pattern) + def references_in(text, pattern = object_reference_pattern) text.gsub(pattern) do |match| yield match, $~[:commit], $~[:project], $~[:namespace], $~ end @@ -22,7 +19,7 @@ module Banzai def find_object(project, id) return unless project.is_a?(Project) && project.valid_repo? - _, record = records_per_parent[project].detect { |k, _v| Gitlab::Git.shas_eql?(k, id) } + _, record = reference_cache.records_per_parent[project].detect { |k, _v| Gitlab::Git.shas_eql?(k, id) } record end @@ -31,7 +28,7 @@ module Banzai return [] unless noteable.is_a?(MergeRequest) @referenced_merge_request_commit_shas ||= begin - referenced_shas = references_per_parent.values.reduce(:|).to_a + referenced_shas = reference_cache.references_per_parent.values.reduce(:|).to_a noteable.all_commit_shas.select do |sha| referenced_shas.any? { |ref| Gitlab::Git.shas_eql?(sha, ref) } end @@ -39,7 +36,7 @@ module Banzai end # The default behaviour is `#to_i` - we just pass the hash through. - def self.parse_symbol(sha_hash, _match) + def parse_symbol(sha_hash, _match) sha_hash end @@ -69,12 +66,12 @@ module Banzai extras end - private - def parent_records(parent, ids) parent.commits_by(oids: ids.to_a) end + private + def noteable context[:noteable] end diff --git a/lib/banzai/filter/references/design_reference_filter.rb b/lib/banzai/filter/references/design_reference_filter.rb index 61234e61c15..01e1036dcec 100644 --- a/lib/banzai/filter/references/design_reference_filter.rb +++ b/lib/banzai/filter/references/design_reference_filter.rb @@ -33,9 +33,10 @@ module Banzai end self.reference_type = :design + self.object_class = ::DesignManagement::Design def find_object(project, identifier) - records_per_parent[project][identifier] + reference_cache.records_per_parent[project][identifier] end def parent_records(project, identifiers) @@ -58,15 +59,6 @@ module Banzai super.includes(:route, :namespace, :group) end - def parent_type - :project - end - - # optimisation to reuse the parent_per_reference query information - def parent_from_ref(ref) - parent_per_reference[ref || current_parent_path] - end - def url_for_object(design, project) path_options = { vueroute: design.filename } Gitlab::Routing.url_helpers.designs_project_issue_path(project, design.issue, path_options) @@ -76,15 +68,11 @@ module Banzai super.merge(issue: design.issue_id) end - def self.object_class - ::DesignManagement::Design - end - - def self.object_sym + def object_sym :design end - def self.parse_symbol(raw, match_data) + def parse_symbol(raw, match_data) filename = match_data[:url_filename] iid = match_data[:issue].to_i Identifier.new(filename: CGI.unescape(filename), issue_iid: iid) diff --git a/lib/banzai/filter/references/epic_reference_filter.rb b/lib/banzai/filter/references/epic_reference_filter.rb index 4ee446e5317..e25734c8b0f 100644 --- a/lib/banzai/filter/references/epic_reference_filter.rb +++ b/lib/banzai/filter/references/epic_reference_filter.rb @@ -21,4 +21,4 @@ module Banzai end end -Banzai::Filter::References::EpicReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::EpicReferenceFilter') +Banzai::Filter::References::EpicReferenceFilter.prepend_mod_with('Banzai::Filter::References::EpicReferenceFilter') diff --git a/lib/banzai/filter/references/external_issue_reference_filter.rb b/lib/banzai/filter/references/external_issue_reference_filter.rb index 247e20967df..1061a9917dd 100644 --- a/lib/banzai/filter/references/external_issue_reference_filter.rb +++ b/lib/banzai/filter/references/external_issue_reference_filter.rb @@ -10,10 +10,11 @@ module Banzai # This filter does not support cross-project references. class ExternalIssueReferenceFilter < ReferenceFilter self.reference_type = :external_issue + self.object_class = ExternalIssue # Public: Find `JIRA-123` issue references in text # - # ExternalIssueReferenceFilter.references_in(text, pattern) do |match, issue| + # references_in(text, pattern) do |match, issue| # "<a href=...>##{issue}</a>" # end # @@ -22,7 +23,7 @@ module Banzai # Yields the String match and the String issue reference. # # Returns a String replaced with the return of the block. - def self.references_in(text, pattern) + def references_in(text, pattern = object_reference_pattern) text.gsub(pattern) do |match| yield match, $~[:issue] end @@ -32,27 +33,7 @@ module Banzai # Early return if the project isn't using an external tracker return doc if project.nil? || default_issues_tracker? - ref_pattern = issue_reference_pattern - ref_start_pattern = /\A#{ref_pattern}\z/ - - nodes.each_with_index do |node, index| - if text_node?(node) - replace_text_when_pattern_matches(node, index, ref_pattern) do |content| - issue_link_filter(content) - end - - elsif element_node?(node) - yield_valid_link(node) do |link, inner_html| - if link =~ ref_start_pattern - replace_link_node_with_href(node, index, link) do - issue_link_filter(link, link_content: inner_html) - end - end - end - end - end - - doc + super end private @@ -65,8 +46,8 @@ module Banzai # # Returns a String with `JIRA-123` references replaced with links. All # links have `gfm` and `gfm-issue` class names attached for styling. - def issue_link_filter(text, link_content: nil) - self.class.references_in(text, issue_reference_pattern) do |match, id| + def object_link_filter(text, pattern, link_content: nil, link_reference: false) + references_in(text) do |match, id| url = url_for_issue(id) klass = reference_class(:issue) data = data_attribute(project: project.id, external_issue: id) @@ -97,14 +78,10 @@ module Banzai external_issues_cached(:default_issues_tracker?) end - def issue_reference_pattern + def object_reference_pattern external_issues_cached(:external_issue_reference_pattern) end - def project - context[:project] - end - def issue_title "Issue in #{project.external_issue_tracker.title}" end diff --git a/lib/banzai/filter/references/feature_flag_reference_filter.rb b/lib/banzai/filter/references/feature_flag_reference_filter.rb index be9ded1ff43..0fb2b1b3b24 100644 --- a/lib/banzai/filter/references/feature_flag_reference_filter.rb +++ b/lib/banzai/filter/references/feature_flag_reference_filter.rb @@ -5,12 +5,9 @@ module Banzai module References class FeatureFlagReferenceFilter < IssuableReferenceFilter self.reference_type = :feature_flag + self.object_class = Operations::FeatureFlag - def self.object_class - Operations::FeatureFlag - end - - def self.object_sym + def object_sym :feature_flag end diff --git a/lib/banzai/filter/references/issuable_reference_filter.rb b/lib/banzai/filter/references/issuable_reference_filter.rb index b8ccb926ae9..6349f8542ca 100644 --- a/lib/banzai/filter/references/issuable_reference_filter.rb +++ b/lib/banzai/filter/references/issuable_reference_filter.rb @@ -9,11 +9,7 @@ module Banzai end def find_object(parent, iid) - records_per_parent[parent][iid] - end - - def parent_from_ref(ref) - parent_per_reference[ref || current_parent_path] + reference_cache.records_per_parent[parent][iid] end end end diff --git a/lib/banzai/filter/references/issue_reference_filter.rb b/lib/banzai/filter/references/issue_reference_filter.rb index eacf261b15f..1053501de7b 100644 --- a/lib/banzai/filter/references/issue_reference_filter.rb +++ b/lib/banzai/filter/references/issue_reference_filter.rb @@ -13,10 +13,7 @@ module Banzai # to reference issues from other GitLab projects. class IssueReferenceFilter < IssuableReferenceFilter self.reference_type = :issue - - def self.object_class - Issue - end + self.object_class = Issue def url_for_object(issue, project) return issue_path(issue, project) if only_path? diff --git a/lib/banzai/filter/references/iteration_reference_filter.rb b/lib/banzai/filter/references/iteration_reference_filter.rb index cf3d446147f..591e07013c3 100644 --- a/lib/banzai/filter/references/iteration_reference_filter.rb +++ b/lib/banzai/filter/references/iteration_reference_filter.rb @@ -6,13 +6,10 @@ module Banzai # The actual filter is implemented in the EE mixin class IterationReferenceFilter < AbstractReferenceFilter self.reference_type = :iteration - - def self.object_class - Iteration - end + self.object_class = Iteration end end end end -Banzai::Filter::References::IterationReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::IterationReferenceFilter') +Banzai::Filter::References::IterationReferenceFilter.prepend_mod_with('Banzai::Filter::References::IterationReferenceFilter') diff --git a/lib/banzai/filter/references/label_reference_filter.rb b/lib/banzai/filter/references/label_reference_filter.rb index a6a5eec5d9a..bf6b3e47d3b 100644 --- a/lib/banzai/filter/references/label_reference_filter.rb +++ b/lib/banzai/filter/references/label_reference_filter.rb @@ -6,10 +6,7 @@ module Banzai # HTML filter that replaces label references with links. class LabelReferenceFilter < AbstractReferenceFilter self.reference_type = :label - - def self.object_class - Label - end + self.object_class = Label def find_object(parent_object, id) find_labels(parent_object).find(id) @@ -20,7 +17,7 @@ module Banzai unescaped_html = unescape_html_entities(text).gsub(pattern) do |match| namespace = $~[:namespace] project = $~[:project] - project_path = full_project_path(namespace, project) + project_path = reference_cache.full_project_path(namespace, project) label = find_label_cached(project_path, $~[:label_id], $~[:label_name]) if label @@ -96,7 +93,7 @@ module Banzai parent = project || group if project || full_path_ref?(matches) - project_path = full_project_path(matches[:namespace], matches[:project]) + project_path = reference_cache.full_project_path(matches[:namespace], matches[:project]) parent_from_ref = from_ref_cached(project_path) reference = parent_from_ref.to_human_reference(parent) @@ -129,4 +126,4 @@ module Banzai end end -Banzai::Filter::References::LabelReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::LabelReferenceFilter') +Banzai::Filter::References::LabelReferenceFilter.prepend_mod_with('Banzai::Filter::References::LabelReferenceFilter') diff --git a/lib/banzai/filter/references/merge_request_reference_filter.rb b/lib/banzai/filter/references/merge_request_reference_filter.rb index 872c33f6873..6c5ad83d9ae 100644 --- a/lib/banzai/filter/references/merge_request_reference_filter.rb +++ b/lib/banzai/filter/references/merge_request_reference_filter.rb @@ -9,10 +9,7 @@ module Banzai # This filter supports cross-project references. class MergeRequestReferenceFilter < IssuableReferenceFilter self.reference_type = :merge_request - - def self.object_class - MergeRequest - end + self.object_class = MergeRequest def url_for_object(mr, project) h = Gitlab::Routing.url_helpers diff --git a/lib/banzai/filter/references/milestone_reference_filter.rb b/lib/banzai/filter/references/milestone_reference_filter.rb index 49110194ddc..31a961f3e73 100644 --- a/lib/banzai/filter/references/milestone_reference_filter.rb +++ b/lib/banzai/filter/references/milestone_reference_filter.rb @@ -8,10 +8,7 @@ module Banzai include Gitlab::Utils::StrongMemoize self.reference_type = :milestone - - def self.object_class - Milestone - end + self.object_class = Milestone # Links to project milestones contain the IID, but when we're handling # 'regular' references, we need to use the global ID to disambiguate @@ -70,7 +67,7 @@ module Banzai end def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name) - project_path = full_project_path(namespace_ref, project_ref) + project_path = reference_cache.full_project_path(namespace_ref, project_ref) # Returns group if project is not found by path parent = parent_from_ref(project_path) diff --git a/lib/banzai/filter/references/project_reference_filter.rb b/lib/banzai/filter/references/project_reference_filter.rb index 522c6e0f5f3..678d2aa3468 100644 --- a/lib/banzai/filter/references/project_reference_filter.rb +++ b/lib/banzai/filter/references/project_reference_filter.rb @@ -6,10 +6,11 @@ module Banzai # HTML filter that replaces project references with links. class ProjectReferenceFilter < ReferenceFilter self.reference_type = :project + self.object_class = Project # Public: Find `namespace/project>` project references in text # - # ProjectReferenceFilter.references_in(text) do |match, project| + # references_in(text) do |match, project| # "<a href=...>#{project}></a>" # end # @@ -18,33 +19,16 @@ module Banzai # Yields the String match, and the String project name. # # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Project.markdown_reference_pattern) do |match| + def references_in(text, pattern = object_reference_pattern) + text.gsub(pattern) do |match| yield match, "#{$~[:namespace]}/#{$~[:project]}" end end - def call - ref_pattern = Project.markdown_reference_pattern - ref_pattern_start = /\A#{ref_pattern}\z/ - - nodes.each_with_index do |node, index| - if text_node?(node) - replace_text_when_pattern_matches(node, index, ref_pattern) do |content| - project_link_filter(content) - end - elsif element_node?(node) - yield_valid_link(node) do |link, inner_html| - if link =~ ref_pattern_start - replace_link_node_with_href(node, index, link) do - project_link_filter(link, link_content: inner_html) - end - end - end - end - end + private - doc + def object_reference_pattern + @object_reference_pattern ||= Project.markdown_reference_pattern end # Replace `namespace/project>` project references in text with links to the referenced @@ -55,8 +39,8 @@ module Banzai # # Returns a String with `namespace/project>` references replaced with links. All links # have `gfm` and `gfm-project` class names attached for styling. - def project_link_filter(text, link_content: nil) - self.class.references_in(text) do |match, project_path| + def object_link_filter(text, pattern, link_content: nil, link_reference: false) + references_in(text) do |match, project_path| cached_call(:banzai_url_for_object, match, path: [Project, project_path.downcase]) do if project = projects_hash[project_path.downcase] link_to_project(project, link_content: link_content) || match @@ -92,8 +76,6 @@ module Banzai refs.to_a end - private - def urls Gitlab::Routing.url_helpers end diff --git a/lib/banzai/filter/references/reference_cache.rb b/lib/banzai/filter/references/reference_cache.rb new file mode 100644 index 00000000000..ab0c74e00d9 --- /dev/null +++ b/lib/banzai/filter/references/reference_cache.rb @@ -0,0 +1,178 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + class ReferenceCache + include Gitlab::Utils::StrongMemoize + include RequestStoreReferenceCache + + def initialize(filter, context) + @filter = filter + @context = context + end + + def load_reference_cache(nodes) + load_references_per_parent(nodes) + load_parent_per_reference + load_records_per_parent + + @cache_loaded = true + end + + # Loads all object references (e.g. issue IDs) per + # project/group they belong to. + def load_references_per_parent(nodes) + @references_per_parent ||= {} + + @references_per_parent[parent_type] ||= begin + refs = Hash.new { |hash, key| hash[key] = Set.new } + + nodes.each do |node| + node.to_html.scan(regex) do + path = if parent_type == :project + full_project_path($~[:namespace], $~[:project]) + else + full_group_path($~[:group]) + end + + ident = filter.identifier($~) + refs[path] << ident if ident + end + end + + refs + end + end + + def references_per_parent + @references_per_parent[parent_type] + end + + # Returns a Hash containing referenced projects grouped per their full + # path. + def load_parent_per_reference + @per_reference ||= {} + + @per_reference[parent_type] ||= begin + refs = references_per_parent.keys.to_set + + find_for_paths(refs.to_a).index_by(&:full_path) + end + end + + def parent_per_reference + @per_reference[parent_type] + end + + def load_records_per_parent + @_records_per_project ||= {} + + @_records_per_project[filter.object_class.to_s.underscore] ||= begin + hash = Hash.new { |h, k| h[k] = {} } + + parent_per_reference.each do |path, parent| + record_ids = references_per_parent[path] + + filter.parent_records(parent, record_ids).each do |record| + hash[parent][filter.record_identifier(record)] = record + end + end + + hash + end + end + + def records_per_parent + @_records_per_project[filter.object_class.to_s.underscore] + end + + def relation_for_paths(paths) + klass = parent_type.to_s.camelize.constantize + result = klass.where_full_path_in(paths) + return result if parent_type == :group + + result.includes(namespace: :route) if parent_type == :project + end + + # Returns projects for the given paths. + def find_for_paths(paths) + if Gitlab::SafeRequestStore.active? + cache = refs_cache + to_query = paths - cache.keys + + unless to_query.empty? + records = relation_for_paths(to_query) + + found = [] + records.each do |record| + ref = record.full_path + get_or_set_cache(cache, ref) { record } + found << ref + end + + not_found = to_query - found + not_found.each do |ref| + get_or_set_cache(cache, ref) { nil } + end + end + + cache.slice(*paths).values.compact + else + relation_for_paths(paths) + end + end + + def current_parent_path + strong_memoize(:current_parent_path) do + parent&.full_path + end + end + + def current_project_namespace_path + strong_memoize(:current_project_namespace_path) do + project&.namespace&.full_path + end + end + + def full_project_path(namespace, project_ref) + return current_parent_path unless project_ref + + namespace_ref = namespace || current_project_namespace_path + "#{namespace_ref}/#{project_ref}" + end + + def full_group_path(group_ref) + return current_parent_path unless group_ref + + group_ref + end + + def cache_loaded? + !!@cache_loaded + end + + private + + attr_accessor :filter, :context + + delegate :project, :group, :parent, :parent_type, to: :filter + + def regex + strong_memoize(:regex) do + [ + filter.object_class.link_reference_pattern, + filter.object_class.reference_pattern + ].compact.reduce { |a, b| Regexp.union(a, b) } + end + end + + def refs_cache + Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} + end + end + end + end +end + +Banzai::Filter::References::ReferenceCache.prepend_mod diff --git a/lib/banzai/filter/references/reference_filter.rb b/lib/banzai/filter/references/reference_filter.rb index dd15c43f5d8..58436f4505e 100644 --- a/lib/banzai/filter/references/reference_filter.rb +++ b/lib/banzai/filter/references/reference_filter.rb @@ -16,8 +16,14 @@ module Banzai include OutputSafety class << self + # Implement in child class + # Example: self.reference_type = :merge_request attr_accessor :reference_type + # Implement in child class + # Example: self.object_class = MergeRequest + attr_accessor :object_class + def call(doc, context = nil, result = nil) new(doc, context, result).call_and_update_nodes end @@ -34,6 +40,77 @@ module Banzai with_update_nodes { call } end + def call + ref_pattern_start = /\A#{object_reference_pattern}\z/ + + nodes.each_with_index do |node, index| + if text_node?(node) + replace_text_when_pattern_matches(node, index, object_reference_pattern) do |content| + object_link_filter(content, object_reference_pattern) + end + elsif element_node?(node) + yield_valid_link(node) do |link, inner_html| + if link =~ ref_pattern_start + replace_link_node_with_href(node, index, link) do + object_link_filter(link, object_reference_pattern, link_content: inner_html) + end + end + end + end + end + + doc + end + + # Public: Find references in text (like `!123` for merge requests) + # + # references_in(text) do |match, id, project_ref, matches| + # object = find_object(project_ref, id) + # "<a href=...>#{object.to_reference}</a>" + # end + # + # text - String text to search. + # + # Yields the String match, the Integer referenced object ID, an optional String + # of the external project reference, and all of the matchdata. + # + # Returns a String replaced with the return of the block. + def references_in(text, pattern = object_reference_pattern) + raise NotImplementedError, "#{self.class} must implement method: #{__callee__}" + end + + # Iterates over all <a> and text() nodes in a document. + # + # Nodes are skipped whenever their ancestor is one of the nodes returned + # by `ignore_ancestor_query`. Link tags are not processed if they have a + # "gfm" class or the "href" attribute is empty. + def each_node + return to_enum(__method__) unless block_given? + + doc.xpath(query).each do |node| + yield node + end + end + + # Returns an Array containing all HTML nodes. + def nodes + @nodes ||= each_node.to_a + end + + def object_class + self.class.object_class + end + + def project + context[:project] + end + + def group + context[:group] + end + + private + # Returns a data attribute String to attach to a reference link # # attributes - Hash, where the key becomes the data attribute name and the @@ -69,12 +146,11 @@ module Banzai end end - def project - context[:project] - end - - def group - context[:group] + # Ensure that a :project key exists in context + # + # Note that while the key might exist, its value could be nil! + def validate + needs :project unless skip_project_check? end def user @@ -93,31 +169,6 @@ module Banzai "#{gfm_klass} has-tooltip" end - # Ensure that a :project key exists in context - # - # Note that while the key might exist, its value could be nil! - def validate - needs :project unless skip_project_check? - end - - # Iterates over all <a> and text() nodes in a document. - # - # Nodes are skipped whenever their ancestor is one of the nodes returned - # by `ignore_ancestor_query`. Link tags are not processed if they have a - # "gfm" class or the "href" attribute is empty. - def each_node - return to_enum(__method__) unless block_given? - - doc.xpath(query).each do |node| - yield node - end - end - - # Returns an Array containing all HTML nodes. - def nodes - @nodes ||= each_node.to_a - end - # Yields the link's URL and inner HTML whenever the node is a valid <a> tag. def yield_valid_link(node) link = unescape_link(node.attr('href').to_s) @@ -132,6 +183,14 @@ module Banzai CGI.unescape(href) end + def unescape_html_entities(text) + CGI.unescapeHTML(text.to_s) + end + + def escape_html_entities(text) + CGI.escapeHTML(text.to_s) + end + def replace_text_when_pattern_matches(node, index, pattern) return unless node.text =~ pattern @@ -161,7 +220,21 @@ module Banzai node.is_a?(Nokogiri::XML::Element) end - private + def object_reference_pattern + @object_reference_pattern ||= object_class.reference_pattern + end + + def object_name + @object_name ||= object_class.name.underscore + end + + def object_sym + @object_sym ||= object_name.to_sym + end + + def object_link_filter(text, pattern, link_content: nil, link_reference: false) + raise NotImplementedError, "#{self.class} must implement method: #{__callee__}" + end def query @query ||= %Q{descendant-or-self::text()[not(#{ignore_ancestor_query})] diff --git a/lib/banzai/filter/references/snippet_reference_filter.rb b/lib/banzai/filter/references/snippet_reference_filter.rb index bf7e0f78609..502bfca1ab7 100644 --- a/lib/banzai/filter/references/snippet_reference_filter.rb +++ b/lib/banzai/filter/references/snippet_reference_filter.rb @@ -9,15 +9,16 @@ module Banzai # This filter supports cross-project references. class SnippetReferenceFilter < AbstractReferenceFilter self.reference_type = :snippet + self.object_class = Snippet - def self.object_class - Snippet + def parent_records(project, ids) + return unless project.is_a?(Project) + + project.snippets.where(id: ids.to_a) end def find_object(project, id) - return unless project.is_a?(Project) - - project.snippets.find_by(id: id) + reference_cache.records_per_parent[project][id] end def url_for_object(snippet, project) diff --git a/lib/banzai/filter/references/user_reference_filter.rb b/lib/banzai/filter/references/user_reference_filter.rb index 04665973f51..1709b607c2e 100644 --- a/lib/banzai/filter/references/user_reference_filter.rb +++ b/lib/banzai/filter/references/user_reference_filter.rb @@ -8,10 +8,11 @@ module Banzai # A special `@all` reference is also supported. class UserReferenceFilter < ReferenceFilter self.reference_type = :user + self.object_class = User # Public: Find `@user` user references in text # - # UserReferenceFilter.references_in(text) do |match, username| + # references_in(text) do |match, username| # "<a href=...>@#{user}</a>" # end # @@ -20,8 +21,8 @@ module Banzai # Yields the String match, and the String user name. # # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(User.reference_pattern) do |match| + def references_in(text, pattern = object_reference_pattern) + text.gsub(pattern) do |match| yield match, $~[:user] end end @@ -29,28 +30,11 @@ module Banzai def call return doc if project.nil? && group.nil? && !skip_project_check? - ref_pattern = User.reference_pattern - ref_pattern_start = /\A#{ref_pattern}\z/ - - nodes.each_with_index do |node, index| - if text_node?(node) - replace_text_when_pattern_matches(node, index, ref_pattern) do |content| - user_link_filter(content) - end - elsif element_node?(node) - yield_valid_link(node) do |link, inner_html| - if link =~ ref_pattern_start - replace_link_node_with_href(node, index, link) do - user_link_filter(link, link_content: inner_html) - end - end - end - end - end - - doc + super end + private + # Replace `@user` user references in text with links to the referenced # user's profile page. # @@ -59,8 +43,8 @@ module Banzai # # Returns a String with `@user` references replaced with links. All links # have `gfm` and `gfm-project_member` class names attached for styling. - def user_link_filter(text, link_content: nil) - self.class.references_in(text) do |match, username| + def object_link_filter(text, pattern, link_content: nil, link_reference: false) + references_in(text, pattern) do |match, username| if username == 'all' && !skip_project_check? link_to_all(link_content: link_content) else @@ -100,8 +84,6 @@ module Banzai refs.to_a end - private - def urls Gitlab::Routing.url_helpers end diff --git a/lib/banzai/filter/references/vulnerability_reference_filter.rb b/lib/banzai/filter/references/vulnerability_reference_filter.rb index e5f2408eda4..aaf45304021 100644 --- a/lib/banzai/filter/references/vulnerability_reference_filter.rb +++ b/lib/banzai/filter/references/vulnerability_reference_filter.rb @@ -6,19 +6,10 @@ module Banzai # The actual filter is implemented in the EE mixin class VulnerabilityReferenceFilter < IssuableReferenceFilter self.reference_type = :vulnerability - - def self.object_class - Vulnerability - end - - private - - def project - context[:project] - end + self.object_class = Vulnerability end end end end -Banzai::Filter::References::VulnerabilityReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::VulnerabilityReferenceFilter') +Banzai::Filter::References::VulnerabilityReferenceFilter.prepend_mod_with('Banzai::Filter::References::VulnerabilityReferenceFilter') diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 06dddc74eba..1e84e7e8af3 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -65,4 +65,4 @@ module Banzai end end -Banzai::Filter::SanitizationFilter.prepend_if_ee('EE::Banzai::Filter::SanitizationFilter') +Banzai::Filter::SanitizationFilter.prepend_mod_with('Banzai::Filter::SanitizationFilter') diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 731a2bb4c77..b16ea689d2e 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -38,7 +38,7 @@ module Banzai begin code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, node.text), tag: language) css_classes << " language-#{language}" if language - rescue + rescue StandardError # Gracefully handle syntax highlighter bugs/errors to ensure users can # still access an issue/comment/etc. First, retry with the plain text # filter. If that fails, then just skip this entirely, but that would diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 762371e1418..ceb7547a85d 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -15,8 +15,16 @@ module Banzai def call return doc if context[:system_note] - linkable_attributes.each do |attr| - process_link_to_upload_attr(attr) + if Feature.enabled?(:optimize_linkable_attributes, project, default_enabled: :yaml) + # We exclude processed upload links from the linkable attributes to + # prevent further modifications by RepositoryLinkFilter + linkable_attributes.reject! do |attr| + process_link_to_upload_attr(attr) + end + else + linkable_attributes.each do |attr| + process_link_to_upload_attr(attr) + end end doc diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index 8994cdbed60..34b6ca99e32 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -58,4 +58,4 @@ module Banzai end end -Banzai::IssuableExtractor.prepend_if_ee('EE::Banzai::IssuableExtractor') +Banzai::IssuableExtractor.prepend_mod_with('Banzai::IssuableExtractor') diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 028e3c44dc3..df8151b3296 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -76,4 +76,4 @@ module Banzai end end -Banzai::Pipeline::GfmPipeline.prepend_if_ee('EE::Banzai::Pipeline::GfmPipeline') +Banzai::Pipeline::GfmPipeline.prepend_mod_with('Banzai::Pipeline::GfmPipeline') diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index 32d7126c97d..889574cf6bf 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -33,4 +33,4 @@ module Banzai end end -Banzai::Pipeline::PostProcessPipeline.prepend_if_ee('EE::Banzai::Pipeline::PostProcessPipeline') +Banzai::Pipeline::PostProcessPipeline.prepend_mod_with('Banzai::Pipeline::PostProcessPipeline') diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index 65a5e28b704..0031ccc7011 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -40,4 +40,4 @@ module Banzai end end -Banzai::Pipeline::SingleLinePipeline.prepend_if_ee('EE::Banzai::Pipeline::SingleLinePipeline') +Banzai::Pipeline::SingleLinePipeline.prepend_mod_with('Banzai::Pipeline::SingleLinePipeline') diff --git a/lib/banzai/reference_parser/epic_parser.rb b/lib/banzai/reference_parser/epic_parser.rb index 7e72a260839..862d09934e9 100644 --- a/lib/banzai/reference_parser/epic_parser.rb +++ b/lib/banzai/reference_parser/epic_parser.rb @@ -13,4 +13,4 @@ module Banzai end end -Banzai::ReferenceParser::EpicParser.prepend_if_ee('::EE::Banzai::ReferenceParser::EpicParser') +Banzai::ReferenceParser::EpicParser.prepend_mod_with('Banzai::ReferenceParser::EpicParser') diff --git a/lib/banzai/reference_parser/iteration_parser.rb b/lib/banzai/reference_parser/iteration_parser.rb index 45253fa1977..981354aa8e1 100644 --- a/lib/banzai/reference_parser/iteration_parser.rb +++ b/lib/banzai/reference_parser/iteration_parser.rb @@ -19,4 +19,4 @@ module Banzai end end -Banzai::ReferenceParser::IterationParser.prepend_if_ee('::EE::Banzai::ReferenceParser::IterationParser') +Banzai::ReferenceParser::IterationParser.prepend_mod_with('Banzai::ReferenceParser::IterationParser') diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index d7bf450465e..24bc1a24e09 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -3,6 +3,8 @@ module Banzai module ReferenceParser class MergeRequestParser < IssuableParser + include Gitlab::Utils::StrongMemoize + self.reference_type = :merge_request def records_for_nodes(nodes) @@ -27,6 +29,16 @@ module Banzai self.class.data_attribute ) end + + def can_read_reference?(user, merge_request) + memo = strong_memoize(:can_read_reference) { {} } + + project_id = merge_request.project_id + + return memo[project_id] if memo.key?(project_id) + + memo[project_id] = can?(user, :read_merge_request_iid, merge_request.project) + end end end end diff --git a/lib/banzai/reference_parser/project_parser.rb b/lib/banzai/reference_parser/project_parser.rb index b4e3a55b4f1..6c600508996 100644 --- a/lib/banzai/reference_parser/project_parser.rb +++ b/lib/banzai/reference_parser/project_parser.rb @@ -19,7 +19,7 @@ module Banzai def readable_project_ids_for(user) @project_ids_by_user ||= {} @project_ids_by_user[user] ||= - Project.public_or_visible_to_user(user).where("projects.id IN (?)", @projects_for_nodes.values.map(&:id)).pluck(:id) + Project.public_or_visible_to_user(user).where(projects: { id: @projects_for_nodes.values.map(&:id) }).pluck(:id) end def can_read_reference?(user, ref_project, node) diff --git a/lib/banzai/reference_parser/vulnerability_parser.rb b/lib/banzai/reference_parser/vulnerability_parser.rb index 143f2605927..86b16605d39 100644 --- a/lib/banzai/reference_parser/vulnerability_parser.rb +++ b/lib/banzai/reference_parser/vulnerability_parser.rb @@ -13,4 +13,4 @@ module Banzai end end -Banzai::ReferenceParser::VulnerabilityParser.prepend_if_ee('::EE::Banzai::ReferenceParser::VulnerabilityParser') +Banzai::ReferenceParser::VulnerabilityParser.prepend_mod_with('Banzai::ReferenceParser::VulnerabilityParser') |