Merge branch 'reference-pipeline-and-caching' into 'master'

Implement different Markdown rendering pipelines and cache Markdown

Builds on !1090. 

Related to !1014.

Fixes #2054.

See merge request !1602

37 files changed, 366 insertions, 220 deletions

diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb
index bf33e5b1b1e..330d3342dd1 100644
--- a/lib/gitlab/asciidoc.rb
+++ b/lib/gitlab/asciidoc.rb
@@ -1,14 +1,10 @@
 require 'asciidoctor'
-require 'html/pipeline'
 
 module Gitlab
   # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters
   # the resulting HTML through HTML pipeline filters.
   module Asciidoc
 
-    # Provide autoload paths for filters to prevent a circular dependency error
-    autoload :RelativeLinkFilter, 'gitlab/markdown/relative_link_filter'
-
     DEFAULT_ADOC_ATTRS = [
       'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab',
       'env-gitlab', 'source-highlighter=html-pipeline'
@@ -24,13 +20,11 @@ module Gitlab
     #                 :requested_path
     #                 :ref
     # asciidoc_opts - a Hash of options to pass to the Asciidoctor converter
-    # html_opts     - a Hash of options for HTML output:
-    #                 :xhtml - output XHTML instead of HTML
     #
-    def self.render(input, context, asciidoc_opts = {}, html_opts = {})
-      asciidoc_opts = asciidoc_opts.reverse_merge(
+    def self.render(input, context, asciidoc_opts = {})
+      asciidoc_opts.reverse_merge!(
         safe: :secure,
-        backend: html_opts[:xhtml] ? :xhtml5 : :html5,
+        backend: :html5,
         attributes: []
       )
       asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS)
@@ -38,23 +32,10 @@ module Gitlab
       html = ::Asciidoctor.convert(input, asciidoc_opts)
 
       if context[:project]
-        result = HTML::Pipeline.new(filters).call(html, context)
-
-        save_opts = html_opts[:xhtml] ?
-          Nokogiri::XML::Node::SaveOptions::AS_XHTML : 0
-
-        html = result[:output].to_html(save_with: save_opts)
+        html = Gitlab::Markdown.render(html, context.merge(pipeline: :asciidoc))
       end
 
       html.html_safe
     end
-
-    private
-
-    def self.filters
-      [
-        Gitlab::Markdown::RelativeLinkFilter
-      ]
-    end
   end
 end
diff --git a/lib/gitlab/markdown.rb b/lib/gitlab/markdown.rb
index 886a09f52af..f4e2cefca51 100644
--- a/lib/gitlab/markdown.rb
+++ b/lib/gitlab/markdown.rb
@@ -19,24 +19,21 @@ module Gitlab
     # context  - Hash of context options passed to our HTML Pipeline
     #
     # Returns an HTML-safe String
-    def self.render(markdown, context = {})
-      html = renderer.render(markdown)
-      html = gfm(html, context)
-
-      html.html_safe
+    def self.render(text, context = {})
+      cache_key = context.delete(:cache_key)
+      cache_key = full_cache_key(cache_key, context[:pipeline])
+
+      if cache_key
+        Rails.cache.fetch(cache_key) do
+          cacheless_render(text, context)
+        end
+      else
+        cacheless_render(text, context)
+      end
     end
 
-    # Convert a Markdown String into HTML without going through the HTML
-    # Pipeline.
-    #
-    # Note that because the pipeline is skipped, SanitizationFilter is as well.
-    # Do not output the result of this method to the user.
-    #
-    # markdown - Markdown String
-    #
-    # Returns a String
-    def self.render_without_gfm(markdown)
-      renderer.render(markdown)
+    def self.render_result(text, context = {})
+      Pipeline[context[:pipeline]].call(text, context)
     end
 
     # Perform post-processing on an HTML String
@@ -46,156 +43,73 @@ module Gitlab
     # permission to make (`RedactorFilter`).
     #
     # html     - String to process
-    # options  - Hash of options to customize output
+    # context  - Hash of options to customize output
     #            :pipeline  - Symbol pipeline type
     #            :project   - Project
     #            :user      - User object
     #
     # Returns an HTML-safe String
-    def self.post_process(html, options)
-      context = {
-        project:      options[:project],
-        current_user: options[:user]
-      }
-      doc = post_processor.to_document(html, context)
+    def self.post_process(html, context)
+      context = Pipeline[context[:pipeline]].transform_context(context)
 
-      if options[:pipeline] == :atom
-        doc.to_html(save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML)
+      pipeline = Pipeline[:post_process]
+      if context[:xhtml]
+        pipeline.to_document(html, context).to_html(save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML)
       else
-        doc.to_html
+        pipeline.to_html(html, context)
       end.html_safe
     end
 
-    # Provide autoload paths for filters to prevent a circular dependency error
-    autoload :AutolinkFilter,               'gitlab/markdown/autolink_filter'
-    autoload :CommitRangeReferenceFilter,   'gitlab/markdown/commit_range_reference_filter'
-    autoload :CommitReferenceFilter,        'gitlab/markdown/commit_reference_filter'
-    autoload :EmojiFilter,                  'gitlab/markdown/emoji_filter'
-    autoload :ExternalIssueReferenceFilter, 'gitlab/markdown/external_issue_reference_filter'
-    autoload :ExternalLinkFilter,           'gitlab/markdown/external_link_filter'
-    autoload :IssueReferenceFilter,         'gitlab/markdown/issue_reference_filter'
-    autoload :LabelReferenceFilter,         'gitlab/markdown/label_reference_filter'
-    autoload :MergeRequestReferenceFilter,  'gitlab/markdown/merge_request_reference_filter'
-    autoload :RedactorFilter,               'gitlab/markdown/redactor_filter'
-    autoload :RelativeLinkFilter,           'gitlab/markdown/relative_link_filter'
-    autoload :SanitizationFilter,           'gitlab/markdown/sanitization_filter'
-    autoload :SnippetReferenceFilter,       'gitlab/markdown/snippet_reference_filter'
-    autoload :SyntaxHighlightFilter,        'gitlab/markdown/syntax_highlight_filter'
-    autoload :TableOfContentsFilter,        'gitlab/markdown/table_of_contents_filter'
-    autoload :TaskListFilter,               'gitlab/markdown/task_list_filter'
-    autoload :UserReferenceFilter,          'gitlab/markdown/user_reference_filter'
-    autoload :UploadLinkFilter,             'gitlab/markdown/upload_link_filter'
-
-    # Public: Parse the provided HTML with GitLab-Flavored Markdown
-    #
-    # html    - HTML String
-    # options - A Hash of options used to customize output (default: {})
-    #           :no_header_anchors - Disable header anchors in TableOfContentsFilter
-    #           :path              - Current path String
-    #           :pipeline          - Symbol pipeline type
-    #           :project           - Current Project object
-    #           :project_wiki      - Current ProjectWiki object
-    #           :ref               - Current ref String
-    #
-    # Returns an HTML-safe String
-    def self.gfm(html, options = {})
-      return '' unless html.present?
-
-      @pipeline ||= HTML::Pipeline.new(filters)
-
-      context = {
-        # SanitizationFilter
-        pipeline: options[:pipeline],
-
-        # EmojiFilter
-        asset_host: Gitlab::Application.config.asset_host,
-        asset_root: Gitlab.config.gitlab.base_url,
-
-        # ReferenceFilter
-        only_path: only_path_pipeline?(options[:pipeline]),
-        project:   options[:project],
-
-        # RelativeLinkFilter
-        project_wiki:   options[:project_wiki],
-        ref:            options[:ref],
-        requested_path: options[:path],
-
-        # TableOfContentsFilter
-        no_header_anchors: options[:no_header_anchors]
-      }
-
-      @pipeline.to_html(html, context).html_safe
-    end
-
     private
 
-    # Check if a pipeline enables the `only_path` context option
-    #
-    # Returns Boolean
-    def self.only_path_pipeline?(pipeline)
-      case pipeline
-      when :atom, :email
-        false
-      else
-        true
-      end
-    end
-
-    def self.redcarpet_options
-      # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use
-      @redcarpet_options ||= {
-        fenced_code_blocks:  true,
-        footnotes:           true,
-        lax_spacing:         true,
-        no_intra_emphasis:   true,
-        space_after_headers: true,
-        strikethrough:       true,
-        superscript:         true,
-        tables:              true
-      }.freeze
-    end
+    def self.cacheless_render(text, context = {})
+      result = render_result(text, context)
 
-    def self.renderer
-      @markdown ||= begin
-        renderer = Redcarpet::Render::HTML.new
-        Redcarpet::Markdown.new(renderer, redcarpet_options)
+      output = result[:output]
+      if output.respond_to?(:to_html)
+        output.to_html
+      else
+        output.to_s
       end
     end
 
-    def self.post_processor
-      @post_processor ||= HTML::Pipeline.new([Gitlab::Markdown::RedactorFilter])
+    def self.full_cache_key(cache_key, pipeline_name)
+      return unless cache_key
+      ["markdown", *cache_key, pipeline_name || :full]
     end
 
-    # Filters used in our pipeline
-    #
-    # SanitizationFilter should come first so that all generated reference HTML
-    # goes through untouched.
-    #
-    # See https://github.com/jch/html-pipeline#filters for more filters.
-    def self.filters
-      [
-        Gitlab::Markdown::SyntaxHighlightFilter,
-        Gitlab::Markdown::SanitizationFilter,
-
-        Gitlab::Markdown::UploadLinkFilter,
-        Gitlab::Markdown::EmojiFilter,
-        Gitlab::Markdown::TableOfContentsFilter,
-        Gitlab::Markdown::AutolinkFilter,
-        Gitlab::Markdown::ExternalLinkFilter,
-
-        Gitlab::Markdown::UserReferenceFilter,
-        Gitlab::Markdown::IssueReferenceFilter,
-        Gitlab::Markdown::ExternalIssueReferenceFilter,
-        Gitlab::Markdown::MergeRequestReferenceFilter,
-        Gitlab::Markdown::SnippetReferenceFilter,
-        Gitlab::Markdown::CommitRangeReferenceFilter,
-        Gitlab::Markdown::CommitReferenceFilter,
-        Gitlab::Markdown::LabelReferenceFilter,
-
-        Gitlab::Markdown::RelativeLinkFilter,
-
-        Gitlab::Markdown::TaskListFilter
-      ]
-    end
+    # Provide autoload paths for filters to prevent a circular dependency error
+    autoload :AutolinkFilter,               'gitlab/markdown/filter/autolink_filter'
+    autoload :CommitRangeReferenceFilter,   'gitlab/markdown/filter/commit_range_reference_filter'
+    autoload :CommitReferenceFilter,        'gitlab/markdown/filter/commit_reference_filter'
+    autoload :EmojiFilter,                  'gitlab/markdown/filter/emoji_filter'
+    autoload :ExternalIssueReferenceFilter, 'gitlab/markdown/filter/external_issue_reference_filter'
+    autoload :ExternalLinkFilter,           'gitlab/markdown/filter/external_link_filter'
+    autoload :IssueReferenceFilter,         'gitlab/markdown/filter/issue_reference_filter'
+    autoload :LabelReferenceFilter,         'gitlab/markdown/filter/label_reference_filter'
+    autoload :MarkdownFilter,               'gitlab/markdown/filter/markdown_filter'
+    autoload :MergeRequestReferenceFilter,  'gitlab/markdown/filter/merge_request_reference_filter'
+    autoload :RedactorFilter,               'gitlab/markdown/filter/redactor_filter'
+    autoload :ReferenceGathererFilter,      'gitlab/markdown/filter/reference_gatherer_filter'
+    autoload :RelativeLinkFilter,           'gitlab/markdown/filter/relative_link_filter'
+    autoload :SanitizationFilter,           'gitlab/markdown/filter/sanitization_filter'
+    autoload :SnippetReferenceFilter,       'gitlab/markdown/filter/snippet_reference_filter'
+    autoload :SyntaxHighlightFilter,        'gitlab/markdown/filter/syntax_highlight_filter'
+    autoload :TableOfContentsFilter,        'gitlab/markdown/filter/table_of_contents_filter'
+    autoload :TaskListFilter,               'gitlab/markdown/filter/task_list_filter'
+    autoload :UserReferenceFilter,          'gitlab/markdown/filter/user_reference_filter'
+    autoload :UploadLinkFilter,             'gitlab/markdown/filter/upload_link_filter'
+
+    autoload :AsciidocPipeline,             'gitlab/markdown/pipeline/asciidoc_pipeline'
+    autoload :AtomPipeline,                 'gitlab/markdown/pipeline/atom_pipeline'
+    autoload :DescriptionPipeline,          'gitlab/markdown/pipeline/description_pipeline'
+    autoload :EmailPipeline,                'gitlab/markdown/pipeline/email_pipeline'
+    autoload :FullPipeline,                 'gitlab/markdown/pipeline/full_pipeline'
+    autoload :GfmPipeline,                  'gitlab/markdown/pipeline/gfm_pipeline'
+    autoload :NotePipeline,                 'gitlab/markdown/pipeline/note_pipeline'
+    autoload :PlainMarkdownPipeline,        'gitlab/markdown/pipeline/plain_markdown_pipeline'
+    autoload :PostProcessPipeline,          'gitlab/markdown/pipeline/post_process_pipeline'
+    autoload :ReferenceExtractionPipeline,  'gitlab/markdown/pipeline/reference_extraction_pipeline'
+    autoload :SingleLinePipeline,           'gitlab/markdown/pipeline/single_line_pipeline'
   end
 end
diff --git a/lib/gitlab/markdown/combined_pipeline.rb b/lib/gitlab/markdown/combined_pipeline.rb
new file mode 100644
index 00000000000..6b08a5e9f72
--- /dev/null
+++ b/lib/gitlab/markdown/combined_pipeline.rb
@@ -0,0 +1,27 @@
+require 'gitlab/markdown'
+
+module Gitlab
+  module Markdown
+    module CombinedPipeline
+      def self.new(*pipelines)
+        Class.new(Pipeline) do
+          const_set :PIPELINES, pipelines
+
+          def self.pipelines
+            self::PIPELINES
+          end
+
+          def self.filters
+            pipelines.flat_map(&:filters)
+          end
+
+          def self.transform_context(context)
+            pipelines.reduce(context) do |context, pipeline|
+              pipeline.transform_context(context)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/autolink_filter.rb b/lib/gitlab/markdown/filter/autolink_filter.rb
index c37c3bc55bf..c37c3bc55bf 100644
--- a/lib/gitlab/markdown/autolink_filter.rb
+++ b/lib/gitlab/markdown/filter/autolink_filter.rb
diff --git a/lib/gitlab/markdown/commit_range_reference_filter.rb b/lib/gitlab/markdown/filter/commit_range_reference_filter.rb
index 36b3258ef76..36b3258ef76 100644
--- a/lib/gitlab/markdown/commit_range_reference_filter.rb
+++ b/lib/gitlab/markdown/filter/commit_range_reference_filter.rb
diff --git a/lib/gitlab/markdown/commit_reference_filter.rb b/lib/gitlab/markdown/filter/commit_reference_filter.rb
index b4036578e60..b4036578e60 100644
--- a/lib/gitlab/markdown/commit_reference_filter.rb
+++ b/lib/gitlab/markdown/filter/commit_reference_filter.rb
diff --git a/lib/gitlab/markdown/emoji_filter.rb b/lib/gitlab/markdown/filter/emoji_filter.rb
index da10e4d3760..da10e4d3760 100644
--- a/lib/gitlab/markdown/emoji_filter.rb
+++ b/lib/gitlab/markdown/filter/emoji_filter.rb
diff --git a/lib/gitlab/markdown/external_issue_reference_filter.rb b/lib/gitlab/markdown/filter/external_issue_reference_filter.rb
index 14bdf5521fc..14bdf5521fc 100644
--- a/lib/gitlab/markdown/external_issue_reference_filter.rb
+++ b/lib/gitlab/markdown/filter/external_issue_reference_filter.rb
diff --git a/lib/gitlab/markdown/external_link_filter.rb b/lib/gitlab/markdown/filter/external_link_filter.rb
index e09dfcb83c8..e09dfcb83c8 100644
--- a/lib/gitlab/markdown/external_link_filter.rb
+++ b/lib/gitlab/markdown/filter/external_link_filter.rb
diff --git a/lib/gitlab/markdown/issue_reference_filter.rb b/lib/gitlab/markdown/filter/issue_reference_filter.rb
index 1ed69e2f431..1ed69e2f431 100644
--- a/lib/gitlab/markdown/issue_reference_filter.rb
+++ b/lib/gitlab/markdown/filter/issue_reference_filter.rb
diff --git a/lib/gitlab/markdown/label_reference_filter.rb b/lib/gitlab/markdown/filter/label_reference_filter.rb
index a2026eecaeb..a2026eecaeb 100644
--- a/lib/gitlab/markdown/label_reference_filter.rb
+++ b/lib/gitlab/markdown/filter/label_reference_filter.rb
diff --git a/lib/gitlab/markdown/filter/markdown_filter.rb b/lib/gitlab/markdown/filter/markdown_filter.rb
new file mode 100644
index 00000000000..921e2a0794e
--- /dev/null
+++ b/lib/gitlab/markdown/filter/markdown_filter.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module Markdown
+    class MarkdownFilter < HTML::Pipeline::TextFilter
+      def initialize(text, context = nil, result = nil)
+        super text, context, result
+        @text = @text.gsub "\r", ''
+      end
+
+      def call
+        html = self.class.renderer.render(@text)
+        html.rstrip!
+        html
+      end
+  
+      private      
+
+      def self.redcarpet_options
+        # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use
+        @redcarpet_options ||= {
+          fenced_code_blocks:  true,
+          footnotes:           true,
+          lax_spacing:         true,
+          no_intra_emphasis:   true,
+          space_after_headers: true,
+          strikethrough:       true,
+          superscript:         true,
+          tables:              true
+        }.freeze
+      end
+
+      def self.renderer
+        @renderer ||= begin
+          renderer = Redcarpet::Render::HTML.new
+          Redcarpet::Markdown.new(renderer, redcarpet_options)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/merge_request_reference_filter.rb b/lib/gitlab/markdown/filter/merge_request_reference_filter.rb
index de71fc76a9b..de71fc76a9b 100644
--- a/lib/gitlab/markdown/merge_request_reference_filter.rb
+++ b/lib/gitlab/markdown/filter/merge_request_reference_filter.rb
diff --git a/lib/gitlab/markdown/redactor_filter.rb b/lib/gitlab/markdown/filter/redactor_filter.rb
index bea714a01e7..33ef7ce18b5 100644
--- a/lib/gitlab/markdown/redactor_filter.rb
+++ b/lib/gitlab/markdown/filter/redactor_filter.rb
@@ -27,7 +27,7 @@ module Gitlab
       def user_can_reference?(node)
         if node.has_attribute?('data-reference-filter')
           reference_type = node.attr('data-reference-filter')
-          reference_filter = reference_type.constantize
+          reference_filter = Gitlab::Markdown.const_get(reference_type)
 
           reference_filter.user_can_reference?(current_user, node, context)
         else
diff --git a/lib/gitlab/markdown/reference_gatherer_filter.rb b/lib/gitlab/markdown/filter/reference_gatherer_filter.rb
index 00f983675e6..62f241b4967 100644
--- a/lib/gitlab/markdown/reference_gatherer_filter.rb
+++ b/lib/gitlab/markdown/filter/reference_gatherer_filter.rb
@@ -31,7 +31,7 @@ module Gitlab
         return unless node.has_attribute?('data-reference-filter')
 
         reference_type = node.attr('data-reference-filter')
-        reference_filter = reference_type.constantize
+        reference_filter = Gitlab::Markdown.const_get(reference_type)
 
         return if context[:reference_filter] && reference_filter != context[:reference_filter]
 
diff --git a/lib/gitlab/markdown/relative_link_filter.rb b/lib/gitlab/markdown/filter/relative_link_filter.rb
index 692c51fd324..81f60120fcd 100644
--- a/lib/gitlab/markdown/relative_link_filter.rb
+++ b/lib/gitlab/markdown/filter/relative_link_filter.rb
@@ -16,10 +16,7 @@ module Gitlab
       def call
         return doc unless linkable_files?
 
-        doc.search('a').each do |el|
-          klass = el.attr('class')
-          next if klass && klass.include?('gfm')
-          
+        doc.search('a:not(.gfm)').each do |el|
           process_link_attr el.attribute('href')
         end
 
diff --git a/lib/gitlab/markdown/sanitization_filter.rb b/lib/gitlab/markdown/filter/sanitization_filter.rb
index ffb9dc33b64..cf153f30622 100644
--- a/lib/gitlab/markdown/sanitization_filter.rb
+++ b/lib/gitlab/markdown/filter/sanitization_filter.rb
@@ -11,7 +11,7 @@ module Gitlab
       def whitelist
         # Descriptions are more heavily sanitized, allowing only a few elements.
         # See http://git.io/vkuAN
-        if pipeline == :description
+        if context[:inline_sanitization]
           whitelist = LIMITED
           whitelist[:elements] -= %w(pre code img ol ul li)
         else
@@ -25,10 +25,6 @@ module Gitlab
 
       private
 
-      def pipeline
-        context[:pipeline] || :default
-      end
-
       def customized?(transformers)
         transformers.last.source_location[0] == __FILE__
       end
diff --git a/lib/gitlab/markdown/snippet_reference_filter.rb b/lib/gitlab/markdown/filter/snippet_reference_filter.rb
index f7bd07c2a34..f7bd07c2a34 100644
--- a/lib/gitlab/markdown/snippet_reference_filter.rb
+++ b/lib/gitlab/markdown/filter/snippet_reference_filter.rb
diff --git a/lib/gitlab/markdown/syntax_highlight_filter.rb b/lib/gitlab/markdown/filter/syntax_highlight_filter.rb
index 8597e02f0de..8597e02f0de 100644
--- a/lib/gitlab/markdown/syntax_highlight_filter.rb
+++ b/lib/gitlab/markdown/filter/syntax_highlight_filter.rb
diff --git a/lib/gitlab/markdown/table_of_contents_filter.rb b/lib/gitlab/markdown/filter/table_of_contents_filter.rb
index bbb3bf7fc8b..bbb3bf7fc8b 100644
--- a/lib/gitlab/markdown/table_of_contents_filter.rb
+++ b/lib/gitlab/markdown/filter/table_of_contents_filter.rb
diff --git a/lib/gitlab/markdown/task_list_filter.rb b/lib/gitlab/markdown/filter/task_list_filter.rb
index 2f133ae8500..2f133ae8500 100644
--- a/lib/gitlab/markdown/task_list_filter.rb
+++ b/lib/gitlab/markdown/filter/task_list_filter.rb
diff --git a/lib/gitlab/markdown/upload_link_filter.rb b/lib/gitlab/markdown/filter/upload_link_filter.rb
index fbada73ab86..fbada73ab86 100644
--- a/lib/gitlab/markdown/upload_link_filter.rb
+++ b/lib/gitlab/markdown/filter/upload_link_filter.rb
diff --git a/lib/gitlab/markdown/user_reference_filter.rb b/lib/gitlab/markdown/filter/user_reference_filter.rb
index 0a20d9c0347..0a20d9c0347 100644
--- a/lib/gitlab/markdown/user_reference_filter.rb
+++ b/lib/gitlab/markdown/filter/user_reference_filter.rb
diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb
new file mode 100644
index 00000000000..d683756f95a
--- /dev/null
+++ b/lib/gitlab/markdown/pipeline.rb
@@ -0,0 +1,34 @@
+require 'gitlab/markdown'
+
+module Gitlab
+  module Markdown
+    class Pipeline
+      def self.[](name)
+        name ||= :full
+        Markdown.const_get("#{name.to_s.camelize}Pipeline")
+      end
+
+      def self.filters
+        []
+      end
+
+      def self.transform_context(context)
+        context
+      end
+
+      def self.html_pipeline
+        @html_pipeline ||= HTML::Pipeline.new(filters)
+      end
+
+      class << self
+        %i(call to_document to_html).each do |meth|
+          define_method(meth) do |text, context|
+            context = transform_context(context)
+
+            html_pipeline.send(meth, text, context)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/pipeline/asciidoc_pipeline.rb b/lib/gitlab/markdown/pipeline/asciidoc_pipeline.rb
new file mode 100644
index 00000000000..6829b4acb95
--- /dev/null
+++ b/lib/gitlab/markdown/pipeline/asciidoc_pipeline.rb
@@ -0,0 +1,13 @@
+require 'gitlab/markdown'
+
+module Gitlab
+  module Markdown
+    class AsciidocPipeline < Pipeline
+      def self.filters
+        [
+          Gitlab::Markdown::RelativeLinkFilter
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/pipeline/atom_pipeline.rb b/lib/gitlab/markdown/pipeline/atom_pipeline.rb
new file mode 100644
index 00000000000..e151f8f5e5a
--- /dev/null
+++ b/lib/gitlab/markdown/pipeline/atom_pipeline.rb
@@ -0,0 +1,14 @@
+require 'gitlab/markdown'
+
+module Gitlab
+  module Markdown
+    class AtomPipeline < FullPipeline
+      def self.transform_context(context)
+        super(context).merge( 
+          only_path: false, 
+          xhtml: true 
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/pipeline/description_pipeline.rb b/lib/gitlab/markdown/pipeline/description_pipeline.rb
new file mode 100644
index 00000000000..76f6948af8f
--- /dev/null
+++ b/lib/gitlab/markdown/pipeline/description_pipeline.rb
@@ -0,0 +1,14 @@
+require 'gitlab/markdown'
+
+module Gitlab
+  module Markdown
+    class DescriptionPipeline < FullPipeline
+      def self.transform_context(context)
+        super(context).merge( 
+          # SanitizationFilter
+          inline_sanitization: true
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/pipeline/email_pipeline.rb b/lib/gitlab/markdown/pipeline/email_pipeline.rb
new file mode 100644
index 00000000000..b88cb790270
--- /dev/null
+++ b/lib/gitlab/markdown/pipeline/email_pipeline.rb
@@ -0,0 +1,13 @@
+require 'gitlab/markdown'
+
+module Gitlab
+  module Markdown
+    class EmailPipeline < FullPipeline
+      def self.transform_context(context)
+        super(context).merge( 
+          only_path: false
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/pipeline/full_pipeline.rb b/lib/gitlab/markdown/pipeline/full_pipeline.rb
new file mode 100644
index 00000000000..b3b7a3c27c0
--- /dev/null
+++ b/lib/gitlab/markdown/pipeline/full_pipeline.rb
@@ -0,0 +1,9 @@
+require 'gitlab/markdown'
+
+module Gitlab
+  module Markdown
+    class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline)
+
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/pipeline/gfm_pipeline.rb b/lib/gitlab/markdown/pipeline/gfm_pipeline.rb
new file mode 100644
index 00000000000..ca90bd75d77
--- /dev/null
+++ b/lib/gitlab/markdown/pipeline/gfm_pipeline.rb
@@ -0,0 +1,41 @@
+require 'gitlab/markdown'
+
+module Gitlab
+  module Markdown
+    class GfmPipeline < Pipeline
+      def self.filters
+        @filters ||= [
+          Gitlab::Markdown::SyntaxHighlightFilter,
+          Gitlab::Markdown::SanitizationFilter,
+
+          Gitlab::Markdown::UploadLinkFilter,
+          Gitlab::Markdown::EmojiFilter,
+          Gitlab::Markdown::TableOfContentsFilter,
+          Gitlab::Markdown::AutolinkFilter,
+          Gitlab::Markdown::ExternalLinkFilter,
+
+          Gitlab::Markdown::UserReferenceFilter,
+          Gitlab::Markdown::IssueReferenceFilter,
+          Gitlab::Markdown::ExternalIssueReferenceFilter,
+          Gitlab::Markdown::MergeRequestReferenceFilter,
+          Gitlab::Markdown::SnippetReferenceFilter,
+          Gitlab::Markdown::CommitRangeReferenceFilter,
+          Gitlab::Markdown::CommitReferenceFilter,
+          Gitlab::Markdown::LabelReferenceFilter,
+
+          Gitlab::Markdown::TaskListFilter
+        ]
+      end
+
+      def self.transform_context(context)
+        context.merge(
+          only_path: true,
+
+          # EmojiFilter
+          asset_host: Gitlab::Application.config.asset_host,
+          asset_root: Gitlab.config.gitlab.base_url
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/pipeline/note_pipeline.rb b/lib/gitlab/markdown/pipeline/note_pipeline.rb
new file mode 100644
index 00000000000..a8bf5f42d8e
--- /dev/null
+++ b/lib/gitlab/markdown/pipeline/note_pipeline.rb
@@ -0,0 +1,14 @@
+require 'gitlab/markdown'
+
+module Gitlab
+  module Markdown
+    class NotePipeline < FullPipeline
+      def self.transform_context(context)
+        super(context).merge( 
+          # TableOfContentsFilter
+          no_header_anchors: true
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/pipeline/plain_markdown_pipeline.rb b/lib/gitlab/markdown/pipeline/plain_markdown_pipeline.rb
new file mode 100644
index 00000000000..0abb93f8a03
--- /dev/null
+++ b/lib/gitlab/markdown/pipeline/plain_markdown_pipeline.rb
@@ -0,0 +1,13 @@
+require 'gitlab/markdown'
+
+module Gitlab
+  module Markdown
+    class PlainMarkdownPipeline < Pipeline
+      def self.filters
+        [
+          Gitlab::Markdown::MarkdownFilter
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/pipeline/post_process_pipeline.rb b/lib/gitlab/markdown/pipeline/post_process_pipeline.rb
new file mode 100644
index 00000000000..60cc32f490e
--- /dev/null
+++ b/lib/gitlab/markdown/pipeline/post_process_pipeline.rb
@@ -0,0 +1,20 @@
+require 'gitlab/markdown'
+
+module Gitlab
+  module Markdown
+    class PostProcessPipeline < Pipeline
+      def self.filters
+        [
+          Gitlab::Markdown::RelativeLinkFilter, 
+          Gitlab::Markdown::RedactorFilter
+        ]
+      end
+
+      def self.transform_context(context)
+        context.merge(
+          post_process: true
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/pipeline/reference_extraction_pipeline.rb b/lib/gitlab/markdown/pipeline/reference_extraction_pipeline.rb
new file mode 100644
index 00000000000..a89ab462bac
--- /dev/null
+++ b/lib/gitlab/markdown/pipeline/reference_extraction_pipeline.rb
@@ -0,0 +1,13 @@
+require 'gitlab/markdown'
+
+module Gitlab
+  module Markdown
+    class ReferenceExtractionPipeline < Pipeline
+      def self.filters
+        [
+          Gitlab::Markdown::ReferenceGathererFilter
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/pipeline/single_line_pipeline.rb b/lib/gitlab/markdown/pipeline/single_line_pipeline.rb
new file mode 100644
index 00000000000..2f24927b879
--- /dev/null
+++ b/lib/gitlab/markdown/pipeline/single_line_pipeline.rb
@@ -0,0 +1,9 @@
+require 'gitlab/markdown'
+
+module Gitlab
+  module Markdown
+    class SingleLinePipeline < GfmPipeline
+      
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/reference_filter.rb b/lib/gitlab/markdown/reference_filter.rb
index b6d93e05ec7..3b83b8bd8f8 100644
--- a/lib/gitlab/markdown/reference_filter.rb
+++ b/lib/gitlab/markdown/reference_filter.rb
@@ -29,6 +29,10 @@ module Gitlab
         end
       end
 
+      def self.[](name)
+        Markdown.const_get("#{name.to_s.camelize}ReferenceFilter")
+      end
+
       def self.user_can_reference?(user, node, context)
         if node.has_attribute?('data-project')
           project_id = node.attr('data-project').to_i
@@ -53,14 +57,14 @@ module Gitlab
       # Examples:
       #
       #   data_attribute(project: 1, issue: 2)
-      #   # => "data-reference-filter=\"Gitlab::Markdown::SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\""
+      #   # => "data-reference-filter=\"SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\""
       #
       #   data_attribute(project: 3, merge_request: 4)
-      #   # => "data-reference-filter=\"Gitlab::Markdown::SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\""
+      #   # => "data-reference-filter=\"SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\""
       #
       # Returns a String
       def data_attribute(attributes = {})
-        attributes[:reference_filter] = self.class.name
+        attributes[:reference_filter] = self.class.name.demodulize
         attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{value}") }.join(" ")
       end
 
diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb
index 3c3478a1271..e83167fa7d7 100644
--- a/lib/gitlab/reference_extractor.rb
+++ b/lib/gitlab/reference_extractor.rb
@@ -9,30 +9,23 @@ module Gitlab
       @project = project
       @current_user = current_user
       @load_lazy_references = load_lazy_references
+
+      @texts = []
+      @references = {}
     end
 
-    def analyze(text)
-      references.clear
-      @text = Gitlab::Markdown.render_without_gfm(text)
+    def analyze(text, options = {})
+      @texts << Gitlab::Markdown.render(text, options.merge(project: project))
     end
 
     %i(user label issue merge_request snippet commit commit_range).each do |type|
       define_method("#{type}s") do
-        references[type]
+        @references[type] ||= pipeline_result(type)
       end
     end
 
     private
 
-    def references
-      @references ||= Hash.new do |references, type|
-        type = type.to_sym
-        next references[type] if references.has_key?(type)
-
-        references[type] = pipeline_result(type)
-      end
-    end
-
     # Instantiate and call HTML::Pipeline with a single reference filter type,
     # returning the result
     #
@@ -40,36 +33,24 @@ module Gitlab
     #
     # Returns the results Array for the requested filter type
     def pipeline_result(filter_type)
-      return [] if @text.blank?
-
-      klass  = "#{filter_type.to_s.camelize}ReferenceFilter"
-      filter = Gitlab::Markdown.const_get(klass)
+      filter = Gitlab::Markdown::ReferenceFilter[filter_type]
 
       context = {
-        project: project,
-        current_user: current_user,
+        pipeline: :reference_extraction,
 
-        # We don't actually care about the links generated
-        only_path: true,
-        ignore_blockquotes: true,
+        project:      project,
+        current_user: current_user,
 
         # ReferenceGathererFilter
         load_lazy_references: false,
         reference_filter:     filter
       }
 
-      # We need to autolink first to finds links to referables, and to prevent
-      # numeric anchors to be parsed as issue references.
-      filters = [
-        Gitlab::Markdown::AutolinkFilter,
-        filter,
-        Gitlab::Markdown::ReferenceGathererFilter
-      ]
-
-      pipeline = HTML::Pipeline.new(filters, context)
-      result = pipeline.call(@text)
-
-      values = result[:references][filter_type].uniq
+      values = @texts.flat_map do |html|
+        text_context = context.dup
+        result = Gitlab::Markdown.render_result(html, text_context)
+        result[:references][filter_type]
+      end.uniq
 
       if @load_lazy_references
         values = Gitlab::Markdown::ReferenceFilter::LazyReference.load(values).uniq