diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-20 09:16:11 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-20 09:16:11 +0000 |
commit | edaa33dee2ff2f7ea3fac488d41558eb5f86d68c (patch) | |
tree | 11f143effbfeba52329fb7afbd05e6e2a3790241 /lib/banzai | |
parent | d8a5691316400a0f7ec4f83832698f1988eb27c1 (diff) | |
download | gitlab-ce-edaa33dee2ff2f7ea3fac488d41558eb5f86d68c.tar.gz |
Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42
Diffstat (limited to 'lib/banzai')
-rw-r--r-- | lib/banzai/filter/base_sanitization_filter.rb | 2 | ||||
-rw-r--r-- | lib/banzai/filter/footnote_filter.rb | 62 | ||||
-rw-r--r-- | lib/banzai/filter/markdown_engines/common_mark.rb | 36 | ||||
-rw-r--r-- | lib/banzai/filter/markdown_post_escape_filter.rb | 18 | ||||
-rw-r--r-- | lib/banzai/filter/plantuml_filter.rb | 7 | ||||
-rw-r--r-- | lib/banzai/filter/references/abstract_reference_filter.rb | 2 | ||||
-rw-r--r-- | lib/banzai/filter/sanitization_filter.rb | 19 | ||||
-rw-r--r-- | lib/banzai/filter/syntax_highlight_filter.rb | 14 | ||||
-rw-r--r-- | lib/banzai/reference_parser/merge_request_parser.rb | 2 | ||||
-rw-r--r-- | lib/banzai/renderer/common_mark/html.rb | 21 |
10 files changed, 40 insertions, 143 deletions
diff --git a/lib/banzai/filter/base_sanitization_filter.rb b/lib/banzai/filter/base_sanitization_filter.rb index 7ea32c4b1e7..4e350a59fa0 100644 --- a/lib/banzai/filter/base_sanitization_filter.rb +++ b/lib/banzai/filter/base_sanitization_filter.rb @@ -42,7 +42,7 @@ module Banzai # Allow any protocol in `a` elements # and then remove links with unsafe protocols allowlist[:protocols].delete('a') - allowlist[:transformers].push(self.class.method(:remove_unsafe_links)) + allowlist[:transformers].push(self.class.method(:sanitize_unsafe_links)) # Remove `rel` attribute from `a` elements allowlist[:transformers].push(self.class.remove_rel) diff --git a/lib/banzai/filter/footnote_filter.rb b/lib/banzai/filter/footnote_filter.rb index 00a38f02141..537b7c80d91 100644 --- a/lib/banzai/filter/footnote_filter.rb +++ b/lib/banzai/filter/footnote_filter.rb @@ -7,13 +7,14 @@ module Banzai # Footnotes are supported in CommonMark. However we were stripping # the ids during sanitization. Those are now allowed. # - # Footnotes are numbered the same - the first one has `id=fn1`, the - # second is `id=fn2`, etc. In order to allow footnotes when rendering - # multiple markdown blocks on a page, we need to make each footnote - # reference unique. - # + # Footnotes are numbered as an increasing integer starting at `1`. + # The `id` associated with a footnote is based on the footnote reference + # string. For example, `[^foot]` will generate `id="fn-foot"`. + # In order to allow footnotes when rendering multiple markdown blocks + # on a page, we need to make each footnote reference unique. + # This filter adds a random number to each footnote (the same number - # can be used for a single render). So you get `id=fn1-4335` and `id=fn2-4335`. + # can be used for a single render). So you get `id=fn-1-4335` and `id=fn-foot-4335`. # class FootnoteFilter < HTML::Pipeline::Filter FOOTNOTE_ID_PREFIX = 'fn-' @@ -26,53 +27,24 @@ module Banzai CSS_FOOTNOTE = 'sup > a[data-footnote-ref]' XPATH_FOOTNOTE = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_FOOTNOTE).freeze - # only needed when feature flag use_cmark_renderer is turned off - INTEGER_PATTERN = /\A\d+\z/.freeze - FOOTNOTE_ID_PREFIX_OLD = 'fn' - FOOTNOTE_LINK_ID_PREFIX_OLD = 'fnref' - FOOTNOTE_LI_REFERENCE_PATTERN_OLD = /\A#{FOOTNOTE_ID_PREFIX_OLD}\d+\z/.freeze - FOOTNOTE_LINK_REFERENCE_PATTERN_OLD = /\A#{FOOTNOTE_LINK_ID_PREFIX_OLD}\d+\z/.freeze - FOOTNOTE_START_NUMBER = 1 - CSS_SECTION_OLD = "ol > li[id=#{FOOTNOTE_ID_PREFIX_OLD}#{FOOTNOTE_START_NUMBER}]" - XPATH_SECTION_OLD = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_SECTION_OLD).freeze - def call - if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - # Sanitization stripped off the section class - add it back in - return doc unless section_node = doc.at_xpath(XPATH_SECTION) + # Sanitization stripped off the section class - add it back in + return doc unless section_node = doc.at_xpath(XPATH_SECTION) - section_node.append_class('footnotes') - else - return doc unless first_footnote = doc.at_xpath(XPATH_SECTION_OLD) - return doc unless first_footnote.parent - - first_footnote.parent.wrap('<section class="footnotes">') - end + section_node.append_class('footnotes') rand_suffix = "-#{random_number}" modified_footnotes = {} - xpath_footnote = if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - XPATH_FOOTNOTE - else - Gitlab::Utils::Nokogiri.css_to_xpath('sup > a[id]') - end - - doc.xpath(xpath_footnote).each do |link_node| - if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX) - ref_num.gsub!(/[[:punct:]]/, '\\\\\&') - else - ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX_OLD) - end + doc.xpath(XPATH_FOOTNOTE).each do |link_node| + ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX) + ref_num.gsub!(/[[:punct:]]/, '\\\\\&') - css = Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? "section[data-footnotes] li[id=#{fn_id(ref_num)}]" : "li[id=#{fn_id(ref_num)}]" + css = "section[data-footnotes] li[id=#{fn_id(ref_num)}]" node_xpath = Gitlab::Utils::Nokogiri.css_to_xpath(css) footnote_node = doc.at_xpath(node_xpath) if footnote_node || modified_footnotes[ref_num] - next if Feature.disabled?(:use_cmark_renderer, default_enabled: :yaml) && !INTEGER_PATTERN.match?(ref_num) - link_node[:href] += rand_suffix link_node[:id] += rand_suffix @@ -103,13 +75,11 @@ module Banzai end def fn_id(num) - prefix = Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? FOOTNOTE_ID_PREFIX : FOOTNOTE_ID_PREFIX_OLD - "#{prefix}#{num}" + "#{FOOTNOTE_ID_PREFIX}#{num}" end def fnref_id(num) - prefix = Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? FOOTNOTE_LINK_ID_PREFIX : FOOTNOTE_LINK_ID_PREFIX_OLD - "#{prefix}#{num}" + "#{FOOTNOTE_LINK_ID_PREFIX}#{num}" end end end diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb index dc94e3c925a..cf368e28beb 100644 --- a/lib/banzai/filter/markdown_engines/common_mark.rb +++ b/lib/banzai/filter/markdown_engines/common_mark.rb @@ -4,8 +4,8 @@ # This module is used in Banzai::Filter::MarkdownFilter. # Used gem is `commonmarker` which is a ruby wrapper for libcmark (CommonMark parser) # including GitHub's GFM extensions. +# We now utilize the renderer built in `C`, rather than the ruby based renderer. # Homepage: https://github.com/gjtorikian/commonmarker - module Banzai module Filter module MarkdownEngines @@ -22,57 +22,29 @@ module Banzai :VALIDATE_UTF8 # replace illegal sequences with the replacement character U+FFFD. ].freeze - RENDER_OPTIONS_C = [ + RENDER_OPTIONS = [ :GITHUB_PRE_LANG, # use GitHub-style <pre lang> for fenced code blocks. :FOOTNOTES, # render footnotes. :FULL_INFO_STRING, # include full info strings of code blocks in separate attribute. :UNSAFE # allow raw/custom HTML and unsafe links. ].freeze - # The `:GITHUB_PRE_LANG` option is not used intentionally because - # it renders a fence block with language as `<pre lang="LANG"><code>some code\n</code></pre>` - # while GitLab's syntax is `<pre><code lang="LANG">some code\n</code></pre>`. - # If in the future the syntax is about to be made GitHub-compatible, please, add `:GITHUB_PRE_LANG` render option below - # and remove `code_block` method from `lib/banzai/renderer/common_mark/html.rb`. - RENDER_OPTIONS_RUBY = [ - # as of commonmarker 0.18.0, we need to use :UNSAFE to get the same as the original :DEFAULT - # https://github.com/gjtorikian/commonmarker/pull/81 - :UNSAFE # allow raw/custom HTML and unsafe links. - ].freeze - def initialize(context) @context = context - @renderer = Banzai::Renderer::CommonMark::HTML.new(options: render_options) if Feature.disabled?(:use_cmark_renderer, default_enabled: :yaml) end def render(text) - if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - CommonMarker.render_html(text, render_options, extensions) - else - doc = CommonMarker.render_doc(text, PARSE_OPTIONS, extensions) - - @renderer.render(doc) - end + CommonMarker.render_html(text, render_options, EXTENSIONS) end private - def extensions - if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - EXTENSIONS - else - EXTENSIONS + [ - :tagfilter # strips out several "unsafe" HTML tags from being used: https://github.github.com/gfm/#disallowed-raw-html-extension- - ].freeze - end - end - def render_options @context[:no_sourcepos] ? render_options_no_sourcepos : render_options_sourcepos end def render_options_no_sourcepos - Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? RENDER_OPTIONS_C : RENDER_OPTIONS_RUBY + RENDER_OPTIONS end def render_options_sourcepos diff --git a/lib/banzai/filter/markdown_post_escape_filter.rb b/lib/banzai/filter/markdown_post_escape_filter.rb index b979b7573ae..09ae09a22ae 100644 --- a/lib/banzai/filter/markdown_post_escape_filter.rb +++ b/lib/banzai/filter/markdown_post_escape_filter.rb @@ -8,8 +8,10 @@ module Banzai NOT_LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-((%5C|\\).+?)-#{LITERAL_KEYWORD}}.freeze SPAN_REGEX = %r{<span>(.*?)</span>}.freeze - CSS_A = 'a' - XPATH_A = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_A).freeze + CSS_A = 'a' + XPATH_A = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_A).freeze + CSS_LANG_TAG = 'pre' + XPATH_LANG_TAG = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_LANG_TAG).freeze def call return doc unless result[:escaped_literals] @@ -32,22 +34,12 @@ module Banzai node.attributes['title'].value = node.attributes['title'].value.gsub(SPAN_REGEX, '\1') if node.attributes['title'] end - doc.xpath(lang_tag).each do |node| + doc.xpath(XPATH_LANG_TAG).each do |node| node.attributes['lang'].value = node.attributes['lang'].value.gsub(SPAN_REGEX, '\1') if node.attributes['lang'] end doc end - - private - - def lang_tag - if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - Gitlab::Utils::Nokogiri.css_to_xpath('pre') - else - Gitlab::Utils::Nokogiri.css_to_xpath('code') - end - end end end end diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index 3f160960d23..68a99702d6f 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -25,12 +25,7 @@ module Banzai private def lang_tag - @lang_tag ||= - if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - Gitlab::Utils::Nokogiri.css_to_xpath('pre[lang="plantuml"] > code').freeze - else - Gitlab::Utils::Nokogiri.css_to_xpath('pre > code[lang="plantuml"]').freeze - end + @lang_tag ||= Gitlab::Utils::Nokogiri.css_to_xpath('pre[lang="plantuml"] > code').freeze end def settings diff --git a/lib/banzai/filter/references/abstract_reference_filter.rb b/lib/banzai/filter/references/abstract_reference_filter.rb index 7a23326bafa..a34519799d5 100644 --- a/lib/banzai/filter/references/abstract_reference_filter.rb +++ b/lib/banzai/filter/references/abstract_reference_filter.rb @@ -216,6 +216,8 @@ module Banzai url_for_object_cached(object, parent) end + url.chomp!(matches[:format]) if matches.names.include?("format") + content = link_content || object_link_text(object, matches) link = %(<a href="#{url}" #{data} diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index d5f45ff7689..fe189b1b0c9 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -28,12 +28,10 @@ module Banzai allowlist[:attributes]['li'] = %w[id] allowlist[:transformers].push(self.class.remove_non_footnote_ids) - if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - # Allow section elements with data-footnotes attribute - allowlist[:elements].push('section') - allowlist[:attributes]['section'] = %w(data-footnotes) - allowlist[:attributes]['a'].push('data-footnote-ref', 'data-footnote-backref') - end + # Allow section elements with data-footnotes attribute + allowlist[:elements].push('section') + allowlist[:attributes]['section'] = %w(data-footnotes) + allowlist[:attributes]['a'].push('data-footnote-ref', 'data-footnote-backref') allowlist end @@ -61,13 +59,8 @@ module Banzai return unless node.name == 'a' || node.name == 'li' return unless node.has_attribute?('id') - if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - return if node.name == 'a' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LINK_REFERENCE_PATTERN - return if node.name == 'li' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LI_REFERENCE_PATTERN - else - return if node.name == 'a' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LINK_REFERENCE_PATTERN_OLD - return if node.name == 'li' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LI_REFERENCE_PATTERN_OLD - end + return if node.name == 'a' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LINK_REFERENCE_PATTERN + return if node.name == 'li' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LI_REFERENCE_PATTERN node.remove_attribute('id') end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 9fcfcf4acc4..07f82c98666 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -14,7 +14,7 @@ module Banzai LANG_PARAMS_DELIMITER = ':' LANG_PARAMS_ATTR = 'data-lang-params' - CSS = 'pre:not([data-math-style]):not([data-mermaid-style]):not([data-kroki-style]) > code' + CSS = 'pre:not([data-math-style]):not([data-mermaid-style]):not([data-kroki-style]) > code:only-child' XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze def call @@ -70,11 +70,11 @@ module Banzai private def parse_lang_params(node) - node = node.parent if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) + node = node.parent # Commonmarker's FULL_INFO_STRING render option works with the space delimiter. # But the current behavior of GitLab's markdown renderer is different - it grabs everything as the single - # line, including language and its options. To keep backward compatability, we have to parse the old format and + # line, including language and its options. To keep backward compatibility, we have to parse the old format and # merge with the new one. # # Behaviors before separating language and its parameters: @@ -91,11 +91,7 @@ module Banzai return unless language language, language_params = language.split(LANG_PARAMS_DELIMITER, 2) - - if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - language_params = [node.attr('data-meta'), language_params].compact.join(' ') - end - + language_params = [node.attr('data-meta'), language_params].compact.join(' ') formatted_language_params = format_language_params(language_params) [language, formatted_language_params] @@ -110,8 +106,8 @@ module Banzai (Rouge::Lexer.find(language) || Rouge::Lexers::PlainText).new end + # Replace the parent `pre` element with the entire highlighted block def replace_parent_pre_element(node, highlighted) - # Replace the parent `pre` element with the entire highlighted block node.parent.replace(highlighted) end diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index 1664fa1f9ff..3e28f06b783 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -8,8 +8,6 @@ module Banzai self.reference_type = :merge_request def nodes_visible_to_user(user, nodes) - return super if Feature.disabled?(:optimize_merge_request_parser, user, default_enabled: :yaml) - merge_request_nodes = nodes.select { |node| node.has_attribute?(self.class.data_attribute) } records = projects_for_nodes(merge_request_nodes) diff --git a/lib/banzai/renderer/common_mark/html.rb b/lib/banzai/renderer/common_mark/html.rb deleted file mode 100644 index d9a2d9a9564..00000000000 --- a/lib/banzai/renderer/common_mark/html.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -# Remove this entire file when removing `use_cmark_renderer` feature flag and switching to the CMARK html renderer. -# https://gitlab.com/gitlab-org/gitlab/-/issues/345744 -module Banzai - module Renderer - module CommonMark - class HTML < CommonMarker::HtmlRenderer - def code_block(node) - block do - out("<pre#{sourcepos(node)}><code") - out(' lang="', node.fence_info, '"') if node.fence_info.present? - out('>') - out(escape_html(node.string_content)) - out('</code></pre>') - end - end - end - end - end -end |