Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42

10 files changed, 40 insertions, 143 deletions

diff --git a/lib/banzai/filter/base_sanitization_filter.rb b/lib/banzai/filter/base_sanitization_filter.rb
index 7ea32c4b1e7..4e350a59fa0 100644
--- a/lib/banzai/filter/base_sanitization_filter.rb
+++ b/lib/banzai/filter/base_sanitization_filter.rb
@@ -42,7 +42,7 @@ module Banzai
           # Allow any protocol in `a` elements
           # and then remove links with unsafe protocols
           allowlist[:protocols].delete('a')
-          allowlist[:transformers].push(self.class.method(:remove_unsafe_links))
+          allowlist[:transformers].push(self.class.method(:sanitize_unsafe_links))
 
           # Remove `rel` attribute from `a` elements
           allowlist[:transformers].push(self.class.remove_rel)
diff --git a/lib/banzai/filter/footnote_filter.rb b/lib/banzai/filter/footnote_filter.rb
index 00a38f02141..537b7c80d91 100644
--- a/lib/banzai/filter/footnote_filter.rb
+++ b/lib/banzai/filter/footnote_filter.rb
@@ -7,13 +7,14 @@ module Banzai
     # Footnotes are supported in CommonMark.  However we were stripping
     # the ids during sanitization.  Those are now allowed.
     #
-    # Footnotes are numbered the same - the first one has `id=fn1`, the
-    # second is `id=fn2`, etc.  In order to allow footnotes when rendering
-    # multiple markdown blocks on a page, we need to make each footnote
-    # reference unique.
-    #
+    # Footnotes are numbered as an increasing integer starting at `1`.
+    # The `id` associated with a footnote is based on the footnote reference
+    # string.  For example, `[^foot]` will generate `id="fn-foot"`.
+    # In order to allow footnotes when rendering multiple markdown blocks
+    # on a page, we need to make each footnote reference unique.
+
     # This filter adds a random number to each footnote (the same number
-    # can be used for a single render). So you get `id=fn1-4335` and `id=fn2-4335`.
+    # can be used for a single render). So you get `id=fn-1-4335` and `id=fn-foot-4335`.
     #
     class FootnoteFilter < HTML::Pipeline::Filter
       FOOTNOTE_ID_PREFIX              = 'fn-'
@@ -26,53 +27,24 @@ module Banzai
       CSS_FOOTNOTE   = 'sup > a[data-footnote-ref]'
       XPATH_FOOTNOTE = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_FOOTNOTE).freeze
 
-      # only needed when feature flag use_cmark_renderer is turned off
-      INTEGER_PATTERN                     = /\A\d+\z/.freeze
-      FOOTNOTE_ID_PREFIX_OLD              = 'fn'
-      FOOTNOTE_LINK_ID_PREFIX_OLD         = 'fnref'
-      FOOTNOTE_LI_REFERENCE_PATTERN_OLD   = /\A#{FOOTNOTE_ID_PREFIX_OLD}\d+\z/.freeze
-      FOOTNOTE_LINK_REFERENCE_PATTERN_OLD = /\A#{FOOTNOTE_LINK_ID_PREFIX_OLD}\d+\z/.freeze
-      FOOTNOTE_START_NUMBER               = 1
-      CSS_SECTION_OLD                     = "ol > li[id=#{FOOTNOTE_ID_PREFIX_OLD}#{FOOTNOTE_START_NUMBER}]"
-      XPATH_SECTION_OLD                   = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_SECTION_OLD).freeze
-
       def call
-        if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-          # Sanitization stripped off the section class - add it back in
-          return doc unless section_node = doc.at_xpath(XPATH_SECTION)
+        # Sanitization stripped off the section class - add it back in
+        return doc unless section_node = doc.at_xpath(XPATH_SECTION)
 
-          section_node.append_class('footnotes')
-        else
-          return doc unless first_footnote = doc.at_xpath(XPATH_SECTION_OLD)
-          return doc unless first_footnote.parent
-
-          first_footnote.parent.wrap('<section class="footnotes">')
-        end
+        section_node.append_class('footnotes')
 
         rand_suffix = "-#{random_number}"
         modified_footnotes = {}
 
-        xpath_footnote = if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-                           XPATH_FOOTNOTE
-                         else
-                           Gitlab::Utils::Nokogiri.css_to_xpath('sup > a[id]')
-                         end
-
-        doc.xpath(xpath_footnote).each do |link_node|
-          if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-            ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX)
-            ref_num.gsub!(/[[:punct:]]/, '\\\\\&')
-          else
-            ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX_OLD)
-          end
+        doc.xpath(XPATH_FOOTNOTE).each do |link_node|
+          ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX)
+          ref_num.gsub!(/[[:punct:]]/, '\\\\\&')
 
-          css = Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? "section[data-footnotes] li[id=#{fn_id(ref_num)}]" : "li[id=#{fn_id(ref_num)}]"
+          css = "section[data-footnotes] li[id=#{fn_id(ref_num)}]"
           node_xpath = Gitlab::Utils::Nokogiri.css_to_xpath(css)
           footnote_node = doc.at_xpath(node_xpath)
 
           if footnote_node || modified_footnotes[ref_num]
-            next if Feature.disabled?(:use_cmark_renderer, default_enabled: :yaml) && !INTEGER_PATTERN.match?(ref_num)
-
             link_node[:href] += rand_suffix
             link_node[:id]   += rand_suffix
 
@@ -103,13 +75,11 @@ module Banzai
       end
 
       def fn_id(num)
-        prefix = Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? FOOTNOTE_ID_PREFIX : FOOTNOTE_ID_PREFIX_OLD
-        "#{prefix}#{num}"
+        "#{FOOTNOTE_ID_PREFIX}#{num}"
       end
 
       def fnref_id(num)
-        prefix = Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? FOOTNOTE_LINK_ID_PREFIX : FOOTNOTE_LINK_ID_PREFIX_OLD
-        "#{prefix}#{num}"
+        "#{FOOTNOTE_LINK_ID_PREFIX}#{num}"
       end
     end
   end
diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb
index dc94e3c925a..cf368e28beb 100644
--- a/lib/banzai/filter/markdown_engines/common_mark.rb
+++ b/lib/banzai/filter/markdown_engines/common_mark.rb
@@ -4,8 +4,8 @@
 # This module is used in Banzai::Filter::MarkdownFilter.
 # Used gem is `commonmarker` which is a ruby wrapper for libcmark (CommonMark parser)
 # including GitHub's GFM extensions.
+# We now utilize the renderer built in `C`, rather than the ruby based renderer.
 # Homepage: https://github.com/gjtorikian/commonmarker
-
 module Banzai
   module Filter
     module MarkdownEngines
@@ -22,57 +22,29 @@ module Banzai
           :VALIDATE_UTF8               # replace illegal sequences with the replacement character U+FFFD.
         ].freeze
 
-        RENDER_OPTIONS_C = [
+        RENDER_OPTIONS = [
           :GITHUB_PRE_LANG,  # use GitHub-style <pre lang> for fenced code blocks.
           :FOOTNOTES,        # render footnotes.
           :FULL_INFO_STRING, # include full info strings of code blocks in separate attribute.
           :UNSAFE            # allow raw/custom HTML and unsafe links.
         ].freeze
 
-        # The `:GITHUB_PRE_LANG` option is not used intentionally because
-        # it renders a fence block with language as `<pre lang="LANG"><code>some code\n</code></pre>`
-        # while GitLab's syntax is `<pre><code lang="LANG">some code\n</code></pre>`.
-        # If in the future the syntax is about to be made GitHub-compatible, please, add `:GITHUB_PRE_LANG` render option below
-        # and remove `code_block` method from `lib/banzai/renderer/common_mark/html.rb`.
-        RENDER_OPTIONS_RUBY = [
-          # as of commonmarker 0.18.0, we need to use :UNSAFE to get the same as the original :DEFAULT
-          # https://github.com/gjtorikian/commonmarker/pull/81
-          :UNSAFE # allow raw/custom HTML and unsafe links.
-        ].freeze
-
         def initialize(context)
           @context = context
-          @renderer = Banzai::Renderer::CommonMark::HTML.new(options: render_options) if Feature.disabled?(:use_cmark_renderer, default_enabled: :yaml)
         end
 
         def render(text)
-          if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-            CommonMarker.render_html(text, render_options, extensions)
-          else
-            doc = CommonMarker.render_doc(text, PARSE_OPTIONS, extensions)
-
-            @renderer.render(doc)
-          end
+          CommonMarker.render_html(text, render_options, EXTENSIONS)
         end
 
         private
 
-        def extensions
-          if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-            EXTENSIONS
-          else
-            EXTENSIONS + [
-              :tagfilter # strips out several "unsafe" HTML tags from being used: https://github.github.com/gfm/#disallowed-raw-html-extension-
-            ].freeze
-          end
-        end
-
         def render_options
           @context[:no_sourcepos] ? render_options_no_sourcepos : render_options_sourcepos
         end
 
         def render_options_no_sourcepos
-          Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? RENDER_OPTIONS_C : RENDER_OPTIONS_RUBY
+          RENDER_OPTIONS
         end
 
         def render_options_sourcepos
diff --git a/lib/banzai/filter/markdown_post_escape_filter.rb b/lib/banzai/filter/markdown_post_escape_filter.rb
index b979b7573ae..09ae09a22ae 100644
--- a/lib/banzai/filter/markdown_post_escape_filter.rb
+++ b/lib/banzai/filter/markdown_post_escape_filter.rb
@@ -8,8 +8,10 @@ module Banzai
       NOT_LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-((%5C|\\).+?)-#{LITERAL_KEYWORD}}.freeze
       SPAN_REGEX        = %r{<span>(.*?)</span>}.freeze
 
-      CSS_A   = 'a'
-      XPATH_A = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_A).freeze
+      CSS_A          = 'a'
+      XPATH_A        = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_A).freeze
+      CSS_LANG_TAG   = 'pre'
+      XPATH_LANG_TAG = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_LANG_TAG).freeze
 
       def call
         return doc unless result[:escaped_literals]
@@ -32,22 +34,12 @@ module Banzai
           node.attributes['title'].value = node.attributes['title'].value.gsub(SPAN_REGEX, '\1') if node.attributes['title']
         end
 
-        doc.xpath(lang_tag).each do |node|
+        doc.xpath(XPATH_LANG_TAG).each do |node|
           node.attributes['lang'].value  = node.attributes['lang'].value.gsub(SPAN_REGEX, '\1') if node.attributes['lang']
         end
 
         doc
       end
-
-      private
-
-      def lang_tag
-        if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-          Gitlab::Utils::Nokogiri.css_to_xpath('pre')
-        else
-          Gitlab::Utils::Nokogiri.css_to_xpath('code')
-        end
-      end
     end
   end
 end
diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb
index 3f160960d23..68a99702d6f 100644
--- a/lib/banzai/filter/plantuml_filter.rb
+++ b/lib/banzai/filter/plantuml_filter.rb
@@ -25,12 +25,7 @@ module Banzai
       private
 
       def lang_tag
-        @lang_tag ||=
-          if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-            Gitlab::Utils::Nokogiri.css_to_xpath('pre[lang="plantuml"] > code').freeze
-          else
-            Gitlab::Utils::Nokogiri.css_to_xpath('pre > code[lang="plantuml"]').freeze
-          end
+        @lang_tag ||= Gitlab::Utils::Nokogiri.css_to_xpath('pre[lang="plantuml"] > code').freeze
       end
 
       def settings
diff --git a/lib/banzai/filter/references/abstract_reference_filter.rb b/lib/banzai/filter/references/abstract_reference_filter.rb
index 7a23326bafa..a34519799d5 100644
--- a/lib/banzai/filter/references/abstract_reference_filter.rb
+++ b/lib/banzai/filter/references/abstract_reference_filter.rb
@@ -216,6 +216,8 @@ module Banzai
                   url_for_object_cached(object, parent)
                 end
 
+              url.chomp!(matches[:format]) if matches.names.include?("format")
+
               content = link_content || object_link_text(object, matches)
 
               link = %(<a href="#{url}" #{data}
diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb
index d5f45ff7689..fe189b1b0c9 100644
--- a/lib/banzai/filter/sanitization_filter.rb
+++ b/lib/banzai/filter/sanitization_filter.rb
@@ -28,12 +28,10 @@ module Banzai
         allowlist[:attributes]['li'] = %w[id]
         allowlist[:transformers].push(self.class.remove_non_footnote_ids)
 
-        if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-          # Allow section elements with data-footnotes attribute
-          allowlist[:elements].push('section')
-          allowlist[:attributes]['section'] = %w(data-footnotes)
-          allowlist[:attributes]['a'].push('data-footnote-ref', 'data-footnote-backref')
-        end
+        # Allow section elements with data-footnotes attribute
+        allowlist[:elements].push('section')
+        allowlist[:attributes]['section'] = %w(data-footnotes)
+        allowlist[:attributes]['a'].push('data-footnote-ref', 'data-footnote-backref')
 
         allowlist
       end
@@ -61,13 +59,8 @@ module Banzai
             return unless node.name == 'a' || node.name == 'li'
             return unless node.has_attribute?('id')
 
-            if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-              return if node.name == 'a' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LINK_REFERENCE_PATTERN
-              return if node.name == 'li' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LI_REFERENCE_PATTERN
-            else
-              return if node.name == 'a' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LINK_REFERENCE_PATTERN_OLD
-              return if node.name == 'li' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LI_REFERENCE_PATTERN_OLD
-            end
+            return if node.name == 'a' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LINK_REFERENCE_PATTERN
+            return if node.name == 'li' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LI_REFERENCE_PATTERN
 
             node.remove_attribute('id')
           end
diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb
index 9fcfcf4acc4..07f82c98666 100644
--- a/lib/banzai/filter/syntax_highlight_filter.rb
+++ b/lib/banzai/filter/syntax_highlight_filter.rb
@@ -14,7 +14,7 @@ module Banzai
       LANG_PARAMS_DELIMITER = ':'
       LANG_PARAMS_ATTR = 'data-lang-params'
 
-      CSS   = 'pre:not([data-math-style]):not([data-mermaid-style]):not([data-kroki-style]) > code'
+      CSS   = 'pre:not([data-math-style]):not([data-mermaid-style]):not([data-kroki-style]) > code:only-child'
       XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze
 
       def call
@@ -70,11 +70,11 @@ module Banzai
       private
 
       def parse_lang_params(node)
-        node = node.parent if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
+        node = node.parent
 
         # Commonmarker's FULL_INFO_STRING render option works with the space delimiter.
         # But the current behavior of GitLab's markdown renderer is different - it grabs everything as the single
-        # line, including language and its options. To keep backward compatability, we have to parse the old format and
+        # line, including language and its options. To keep backward compatibility, we have to parse the old format and
         # merge with the new one.
         #
         # Behaviors before separating language and its parameters:
@@ -91,11 +91,7 @@ module Banzai
         return unless language
 
         language, language_params = language.split(LANG_PARAMS_DELIMITER, 2)
-
-        if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-          language_params = [node.attr('data-meta'), language_params].compact.join(' ')
-        end
-
+        language_params = [node.attr('data-meta'), language_params].compact.join(' ')
         formatted_language_params = format_language_params(language_params)
 
         [language, formatted_language_params]
@@ -110,8 +106,8 @@ module Banzai
         (Rouge::Lexer.find(language) || Rouge::Lexers::PlainText).new
       end
 
+      # Replace the parent `pre` element with the entire highlighted block
       def replace_parent_pre_element(node, highlighted)
-        # Replace the parent `pre` element with the entire highlighted block
         node.parent.replace(highlighted)
       end
 
diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb
index 1664fa1f9ff..3e28f06b783 100644
--- a/lib/banzai/reference_parser/merge_request_parser.rb
+++ b/lib/banzai/reference_parser/merge_request_parser.rb
@@ -8,8 +8,6 @@ module Banzai
       self.reference_type = :merge_request
 
       def nodes_visible_to_user(user, nodes)
-        return super if Feature.disabled?(:optimize_merge_request_parser, user, default_enabled: :yaml)
-
         merge_request_nodes = nodes.select { |node| node.has_attribute?(self.class.data_attribute) }
         records = projects_for_nodes(merge_request_nodes)
 
diff --git a/lib/banzai/renderer/common_mark/html.rb b/lib/banzai/renderer/common_mark/html.rb
deleted file mode 100644
index d9a2d9a9564..00000000000
--- a/lib/banzai/renderer/common_mark/html.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-# Remove this entire file when removing `use_cmark_renderer` feature flag and switching to the CMARK html renderer.
-# https://gitlab.com/gitlab-org/gitlab/-/issues/345744
-module Banzai
-  module Renderer
-    module CommonMark
-      class HTML < CommonMarker::HtmlRenderer
-        def code_block(node)
-          block do
-            out("<pre#{sourcepos(node)}><code")
-            out(' lang="', node.fence_info, '"') if node.fence_info.present?
-            out('>')
-            out(escape_html(node.string_content))
-            out('</code></pre>')
-          end
-        end
-      end
-    end
-  end
-end