Add latest changes from gitlab-org/gitlab@12-4-stable-ee

14 files changed, 294 insertions, 89 deletions

diff --git a/lib/banzai/filter.rb b/lib/banzai/filter.rb
index 7d9766c906c..2438cb3c166 100644
--- a/lib/banzai/filter.rb
+++ b/lib/banzai/filter.rb
@@ -3,7 +3,7 @@
 module Banzai
   module Filter
     def self.[](name)
-      const_get("#{name.to_s.camelize}Filter")
+      const_get("#{name.to_s.camelize}Filter", false)
     end
   end
 end
diff --git a/lib/banzai/filter/ascii_doc_sanitization_filter.rb b/lib/banzai/filter/ascii_doc_sanitization_filter.rb
index 9105e86ad04..e41f7d8488a 100644
--- a/lib/banzai/filter/ascii_doc_sanitization_filter.rb
+++ b/lib/banzai/filter/ascii_doc_sanitization_filter.rb
@@ -22,6 +22,10 @@ module Banzai
       CHECKLIST_CLASSES = %w(fa fa-check-square-o fa-square-o).freeze
       LIST_CLASSES = %w(checklist none no-bullet unnumbered unstyled).freeze
 
+      TABLE_FRAME_CLASSES = %w(frame-all frame-topbot frame-sides frame-ends frame-none).freeze
+      TABLE_GRID_CLASSES = %w(grid-all grid-rows grid-cols grid-none).freeze
+      TABLE_STRIPES_CLASSES = %w(stripes-all stripes-odd stripes-even stripes-hover stripes-none).freeze
+
       ELEMENT_CLASSES_WHITELIST = {
         span: %w(big small underline overline line-through).freeze,
         div: ['admonitionblock'].freeze,
@@ -29,7 +33,8 @@ module Banzai
         i: ADMONITION_CLASSES + CALLOUT_CLASSES + CHECKLIST_CLASSES,
         ul: LIST_CLASSES,
         ol: LIST_CLASSES,
-        a: ['anchor'].freeze
+        a: ['anchor'].freeze,
+        table: TABLE_FRAME_CLASSES + TABLE_GRID_CLASSES + TABLE_STRIPES_CLASSES
       }.freeze
 
       def customize_whitelist(whitelist)
@@ -45,6 +50,7 @@ module Banzai
         whitelist[:attributes]['ul'] = %w(class)
         whitelist[:attributes]['ol'] = %w(class)
         whitelist[:attributes]['a'].push('class')
+        whitelist[:attributes]['table'] = %w(class)
         whitelist[:transformers].push(self.class.remove_element_classes)
 
         # Allow `id` in heading elements for section anchors
diff --git a/lib/banzai/filter/audio_link_filter.rb b/lib/banzai/filter/audio_link_filter.rb
new file mode 100644
index 00000000000..50472c3cf81
--- /dev/null
+++ b/lib/banzai/filter/audio_link_filter.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/audio.js
+module Banzai
+  module Filter
+    class AudioLinkFilter < PlayableLinkFilter
+      private
+
+      def media_type
+        "audio"
+      end
+
+      def safe_media_ext
+        Gitlab::FileTypeDetection::SAFE_AUDIO_EXT
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/playable_link_filter.rb b/lib/banzai/filter/playable_link_filter.rb
new file mode 100644
index 00000000000..0a043aa809c
--- /dev/null
+++ b/lib/banzai/filter/playable_link_filter.rb
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Banzai
+  module Filter
+    # Find every image that isn't already wrapped in an `a` tag, and that has
+    # a `src` attribute ending with an audio or video extension, add a new audio or video node and
+    # a "Download" link in the case the media cannot be played.
+    class PlayableLinkFilter < HTML::Pipeline::Filter
+      def call
+        doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |el|
+          el.replace(media_node(doc, el)) if has_media_extension?(el)
+        end
+
+        doc
+      end
+
+      private
+
+      def media_type
+        raise NotImplementedError
+      end
+
+      def safe_media_ext
+        raise NotImplementedError
+      end
+
+      def extra_element_attrs
+        {}
+      end
+
+      def has_media_extension?(element)
+        src = element.attr('data-canonical-src').presence || element.attr('src')
+
+        return unless src.present?
+
+        src_ext = File.extname(src).sub('.', '').downcase
+        safe_media_ext.include?(src_ext)
+      end
+
+      def media_element(doc, element)
+        media_element_attrs = {
+            src: element['src'],
+            controls: true,
+            'data-setup': '{}',
+            'data-title': element['title'] || element['alt']
+        }.merge!(extra_element_attrs)
+
+        if element['data-canonical-src']
+          media_element_attrs['data-canonical-src'] = element['data-canonical-src']
+        end
+
+        doc.document.create_element(media_type, media_element_attrs)
+      end
+
+      def download_paragraph(doc, element)
+        link_content = element['title'] || element['alt']
+
+        link_element_attrs = {
+          href: element['src'],
+          target: '_blank',
+          rel: 'noopener noreferrer',
+          title: "Download '#{link_content}'"
+        }
+
+        # make sure the original non-proxied src carries over
+        if element['data-canonical-src']
+          link_element_attrs['data-canonical-src'] = element['data-canonical-src']
+        end
+
+        link = doc.document.create_element('a', link_content, link_element_attrs)
+
+        doc.document.create_element('p').tap do |paragraph|
+          paragraph.children = link
+        end
+      end
+
+      def media_node(doc, element)
+        container_element_attrs = { class: "#{media_type}-container" }
+
+        doc.document.create_element( "div", container_element_attrs).tap do |container|
+          container.add_child(media_element(doc, element))
+          container.add_child(download_paragraph(doc, element))
+        end
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb
index e8001889ca3..c7589e69262 100644
--- a/lib/banzai/filter/relative_link_filter.rb
+++ b/lib/banzai/filter/relative_link_filter.rb
@@ -20,16 +20,13 @@ module Banzai
       def call
         return doc if context[:system_note]
 
-        @uri_types = {}
         clear_memoization(:linkable_files)
+        clear_memoization(:linkable_attributes)
 
-        doc.search('a:not(.gfm)').each do |el|
-          process_link_attr el.attribute('href')
-        end
+        load_uri_types
 
-        doc.css('img, video').each do |el|
-          process_link_attr el.attribute('src')
-          process_link_attr el.attribute('data-src')
+        linkable_attributes.each do |attr|
+          process_link_attr(attr)
         end
 
         doc
@@ -37,16 +34,80 @@ module Banzai
 
       protected
 
+      def load_uri_types
+        return unless linkable_files?
+        return unless linkable_attributes.present?
+        return {} unless repository
+
+        @uri_types = request_path.present? ? get_uri_types([request_path]) : {}
+
+        paths = linkable_attributes.flat_map do |attr|
+          [get_uri(attr).to_s, relative_file_path(get_uri(attr))]
+        end
+
+        paths.reject!(&:blank?)
+        paths.uniq!
+
+        @uri_types.merge!(get_uri_types(paths))
+      end
+
       def linkable_files?
         strong_memoize(:linkable_files) do
           context[:project_wiki].nil? && repository.try(:exists?) && !repository.empty?
         end
       end
 
-      def process_link_attr(html_attr)
-        return if html_attr.blank?
-        return if html_attr.value.start_with?('//')
+      def linkable_attributes
+        strong_memoize(:linkable_attributes) do
+          attrs = []
+
+          attrs += doc.search('a:not(.gfm)').map do |el|
+            el.attribute('href')
+          end
+
+          attrs += doc.search('img, video, audio').flat_map do |el|
+            [el.attribute('src'), el.attribute('data-src')]
+          end
+
+          attrs.reject do |attr|
+            attr.blank? || attr.value.start_with?('//')
+          end
+        end
+      end
+
+      def get_uri_types(paths)
+        return {} if paths.empty?
+
+        uri_types = Hash[paths.collect { |name| [name, nil] }]
+
+        get_blob_types(paths).each do |name, type|
+          if type == :blob
+            blob = ::Blob.decorate(Gitlab::Git::Blob.new(name: name), project)
+            uri_types[name] = blob.image? || blob.video? || blob.audio? ? :raw : :blob
+          else
+            uri_types[name] = type
+          end
+        end
+
+        uri_types
+      end
 
+      def get_blob_types(paths)
+        revision_paths = paths.collect do |path|
+          [current_commit.sha, path.chomp("/")]
+        end
+
+        Gitlab::GitalyClient::BlobService.new(repository).get_blob_types(revision_paths, 1)
+      end
+
+      def get_uri(html_attr)
+        uri = URI(html_attr.value)
+
+        uri if uri.relative? && uri.path.present?
+      rescue URI::Error, Addressable::URI::InvalidURIError
+      end
+
+      def process_link_attr(html_attr)
         if html_attr.value.start_with?('/uploads/')
           process_link_to_upload_attr(html_attr)
         elsif linkable_files? && repo_visible_to_user?
@@ -81,6 +142,7 @@ module Banzai
 
       def process_link_to_repository_attr(html_attr)
         uri = URI(html_attr.value)
+
         if uri.relative? && uri.path.present?
           html_attr.value = rebuild_relative_uri(uri).to_s
         end
@@ -89,7 +151,7 @@ module Banzai
       end
 
       def rebuild_relative_uri(uri)
-        file_path = relative_file_path(uri)
+        file_path = nested_file_path_if_exists(uri)
 
         uri.path = [
           relative_url_root,
@@ -102,13 +164,29 @@ module Banzai
         uri
       end
 
-      def relative_file_path(uri)
-        path = Addressable::URI.unescape(uri.path).delete("\0")
-        request_path = Addressable::URI.unescape(context[:requested_path])
-        nested_path = build_relative_path(path, request_path)
+      def nested_file_path_if_exists(uri)
+        path = cleaned_file_path(uri)
+        nested_path = relative_file_path(uri)
+
         file_exists?(nested_path) ? nested_path : path
       end
 
+      def cleaned_file_path(uri)
+        Addressable::URI.unescape(uri.path).delete("\0").chomp("/")
+      end
+
+      def relative_file_path(uri)
+        return if uri.nil?
+
+        build_relative_path(cleaned_file_path(uri), request_path)
+      end
+
+      def request_path
+        return unless context[:requested_path]
+
+        Addressable::URI.unescape(context[:requested_path]).chomp("/")
+      end
+
       # Convert a relative path into its correct location based on the currently
       # requested path
       #
@@ -136,6 +214,7 @@ module Banzai
         return path[1..-1] if path.start_with?('/')
 
         parts = request_path.split('/')
+
         parts.pop if uri_type(request_path) != :tree
 
         path.sub!(%r{\A\./}, '')
@@ -149,14 +228,11 @@ module Banzai
       end
 
       def file_exists?(path)
-        path.present? && !!uri_type(path)
+        path.present? && uri_type(path).present?
       end
 
       def uri_type(path)
-        # https://gitlab.com/gitlab-org/gitlab-foss/issues/58657
-        Gitlab::GitalyClient.allow_n_plus_1_calls do
-          @uri_types[path] ||= current_commit.uri_type(path)
-        end
+        @uri_types[path] == :unknown ? "" : @uri_types[path]
       end
 
       def current_commit
diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb
index ade4d260be1..a2c8e92e560 100644
--- a/lib/banzai/filter/table_of_contents_filter.rb
+++ b/lib/banzai/filter/table_of_contents_filter.rb
@@ -56,7 +56,8 @@ module Banzai
       private
 
       def anchor_tag(href)
-        %Q{<a id="user-content-#{href}" class="anchor" href="##{href}" aria-hidden="true"></a>}
+        escaped_href = CGI.escape(href) # account for non-ASCII characters
+        %Q{<a id="user-content-#{href}" class="anchor" href="##{escaped_href}" aria-hidden="true"></a>}
       end
 
       def push_toc(children, root: false)
@@ -80,7 +81,7 @@ module Banzai
 
         def initialize(node: nil, href: nil, previous_header: nil)
           @node = node
-          @href = href
+          @href = CGI.escape(href) if href
           @children = []
 
           @parent = find_parent(previous_header)
diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb
index a278fcfdb47..ed82fbc1f94 100644
--- a/lib/banzai/filter/video_link_filter.rb
+++ b/lib/banzai/filter/video_link_filter.rb
@@ -3,73 +3,19 @@
 # Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/video.js
 module Banzai
   module Filter
-    # Find every image that isn't already wrapped in an `a` tag, and that has
-    # a `src` attribute ending with a video extension, add a new video node and
-    # a "Download" link in the case the video cannot be played.
-    class VideoLinkFilter < HTML::Pipeline::Filter
-      def call
-        doc.xpath(query).each do |el|
-          el.replace(video_node(doc, el))
-        end
-
-        doc
-      end
-
+    class VideoLinkFilter < PlayableLinkFilter
       private
 
-      def query
-        @query ||= begin
-          src_query = UploaderHelper::VIDEO_EXT.map do |ext|
-            "'.#{ext}' = substring(@src, string-length(@src) - #{ext.size})"
-          end
-
-          if context[:asset_proxy_enabled].present?
-            src_query.concat(
-              UploaderHelper::VIDEO_EXT.map do |ext|
-                "'.#{ext}' = substring(@data-canonical-src, string-length(@data-canonical-src) - #{ext.size})"
-              end
-            )
-          end
-
-          "descendant-or-self::img[not(ancestor::a) and (#{src_query.join(' or ')})]"
-        end
+      def media_type
+        "video"
       end
 
-      def video_node(doc, element)
-        container = doc.document.create_element(
-          'div',
-          class: 'video-container'
-        )
-
-        video = doc.document.create_element(
-          'video',
-          src: element['src'],
-          width: '400',
-          controls: true,
-          'data-setup' => '{}',
-          'data-title' => element['title'] || element['alt'])
-
-        link = doc.document.create_element(
-          'a',
-          element['title'] || element['alt'],
-          href: element['src'],
-          target: '_blank',
-          rel: 'noopener noreferrer',
-          title: "Download '#{element['title'] || element['alt']}'")
-
-        # make sure the original non-proxied src carries over
-        if element['data-canonical-src']
-          video['data-canonical-src'] = element['data-canonical-src']
-          link['data-canonical-src']  = element['data-canonical-src']
-        end
-
-        download_paragraph = doc.document.create_element('p')
-        download_paragraph.children = link
-
-        container.add_child(video)
-        container.add_child(download_paragraph)
+      def safe_media_ext
+        Gitlab::FileTypeDetection::SAFE_VIDEO_EXT
+      end
 
-        container
+      def extra_element_attrs
+        { width: "100%" }
       end
     end
   end
diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb
index 18947679b69..205f777bc90 100644
--- a/lib/banzai/filter/wiki_link_filter.rb
+++ b/lib/banzai/filter/wiki_link_filter.rb
@@ -15,7 +15,7 @@ module Banzai
 
         doc.search('a:not(.gfm)').each { |el| process_link(el.attribute('href'), el) }
 
-        doc.search('video').each { |el| process_link(el.attribute('src'), el) }
+        doc.search('video, audio').each { |el| process_link(el.attribute('src'), el) }
 
         doc.search('img').each do |el|
           attr = el.attribute('data-src') || el.attribute('src')
diff --git a/lib/banzai/pipeline.rb b/lib/banzai/pipeline.rb
index e8a81bebaa9..497d3f27542 100644
--- a/lib/banzai/pipeline.rb
+++ b/lib/banzai/pipeline.rb
@@ -4,7 +4,7 @@ module Banzai
   module Pipeline
     def self.[](name)
       name ||= :full
-      const_get("#{name.to_s.camelize}Pipeline")
+      const_get("#{name.to_s.camelize}Pipeline", false)
     end
   end
 end
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index bb0d1eaa1e1..08e27257fdf 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -26,6 +26,7 @@ module Banzai
           Filter::ColorFilter,
           Filter::MermaidFilter,
           Filter::VideoLinkFilter,
+          Filter::AudioLinkFilter,
           Filter::ImageLazyLoadFilter,
           Filter::ImageLinkFilter,
           Filter::InlineMetricsFilter,
diff --git a/lib/banzai/reference_parser.rb b/lib/banzai/reference_parser.rb
index efe15096f08..c08d3364a87 100644
--- a/lib/banzai/reference_parser.rb
+++ b/lib/banzai/reference_parser.rb
@@ -10,7 +10,7 @@ module Banzai
     #
     # This would return the `Banzai::ReferenceParser::IssueParser` class.
     def self.[](name)
-      const_get("#{name.to_s.camelize}Parser")
+      const_get("#{name.to_s.camelize}Parser", false)
     end
   end
 end
diff --git a/lib/banzai/reference_parser/mentioned_user_parser.rb b/lib/banzai/reference_parser/mentioned_user_parser.rb
new file mode 100644
index 00000000000..4b1bcb3ca09
--- /dev/null
+++ b/lib/banzai/reference_parser/mentioned_user_parser.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Banzai
+  module ReferenceParser
+    class MentionedUserParser < BaseParser
+      self.reference_type = :user
+
+      def references_relation
+        User
+      end
+
+      # any user can be mentioned by username
+      def can_read_reference?(user, ref_attr, node)
+        true
+      end
+    end
+  end
+end
diff --git a/lib/banzai/reference_parser/mentioned_users_by_group_parser.rb b/lib/banzai/reference_parser/mentioned_users_by_group_parser.rb
new file mode 100644
index 00000000000..d4ff6a12cd0
--- /dev/null
+++ b/lib/banzai/reference_parser/mentioned_users_by_group_parser.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Banzai
+  module ReferenceParser
+    class MentionedUsersByGroupParser < BaseParser
+      GROUP_ATTR = 'data-group'
+
+      self.reference_type = :user
+
+      def self.data_attribute
+        @data_attribute ||= GROUP_ATTR
+      end
+
+      def references_relation
+        Group
+      end
+
+      def nodes_visible_to_user(user, nodes)
+        groups = lazy { grouped_objects_for_nodes(nodes, Group, GROUP_ATTR) }
+
+        nodes.select do |node|
+          node.has_attribute?(GROUP_ATTR) && can_read_group_reference?(node, user, groups)
+        end
+      end
+
+      def can_read_group_reference?(node, user, groups)
+        node_group = groups[node]
+
+        node_group && can?(user, :read_group, node_group)
+      end
+    end
+  end
+end
diff --git a/lib/banzai/reference_parser/mentioned_users_by_project_parser.rb b/lib/banzai/reference_parser/mentioned_users_by_project_parser.rb
new file mode 100644
index 00000000000..79258d81cc3
--- /dev/null
+++ b/lib/banzai/reference_parser/mentioned_users_by_project_parser.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Banzai
+  module ReferenceParser
+    class MentionedUsersByProjectParser < ProjectParser
+      PROJECT_ATTR = 'data-project'
+
+      self.reference_type = :user
+
+      def self.data_attribute
+        @data_attribute ||= PROJECT_ATTR
+      end
+
+      def references_relation
+        Project
+      end
+    end
+  end
+end