diff options
| -rw-r--r-- | Gemfile | 1 | ||||
| -rw-r--r-- | Gemfile.lock | 5 | ||||
| -rw-r--r-- | changelogs/unreleased/replace_redcarpet_with_cmark.yml | 5 | ||||
| -rw-r--r-- | config/initializers/8_metrics.rb | 1 | ||||
| -rw-r--r-- | lib/banzai/filter/markdown_engines/common_mark.rb | 45 | ||||
| -rw-r--r-- | lib/banzai/filter/markdown_engines/redcarpet.rb | 32 | ||||
| -rw-r--r-- | lib/banzai/filter/markdown_filter.rb | 41 | ||||
| -rw-r--r-- | lib/banzai/filter/syntax_highlight_filter.rb | 1 | ||||
| -rw-r--r-- | lib/banzai/renderer/common_mark/html.rb | 21 | ||||
| -rw-r--r-- | lib/banzai/renderer/html.rb | 13 | ||||
| -rw-r--r-- | lib/banzai/renderer/redcarpet/html.rb | 15 | ||||
| -rw-r--r-- | lib/rouge/plugins/common_mark.rb | 20 |
12 files changed, 165 insertions, 35 deletions
@@ -126,6 +126,7 @@ gem 'html-pipeline', '~> 1.11.0' gem 'deckar01-task_list', '2.0.0' gem 'gitlab-markup', '~> 1.6.2' gem 'redcarpet', '~> 3.4' +gem 'commonmarker', '~> 0.17' gem 'RedCloth', '~> 4.3.2' gem 'rdoc', '~> 4.2' gem 'org-ruby', '~> 0.9.12' diff --git a/Gemfile.lock b/Gemfile.lock index 89b86ae0259..edb2d887c0d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -131,6 +131,8 @@ GEM coercible (1.0.0) descendants_tracker (~> 0.0.1) colorize (0.7.7) + commonmarker (0.17.8) + ruby-enum (~> 0.5) concord (0.1.5) adamantium (~> 0.2.0) equalizer (~> 0.0.9) @@ -797,6 +799,8 @@ GEM rubocop (>= 0.51) rubocop-rspec (1.22.1) rubocop (>= 0.52.1) + ruby-enum (0.7.2) + i18n ruby-fogbugz (0.2.1) crack (~> 0.4) ruby-prof (0.16.2) @@ -1019,6 +1023,7 @@ DEPENDENCIES charlock_holmes (~> 0.7.5) chronic (~> 0.10.2) chronic_duration (~> 0.10.6) + commonmarker (~> 0.17) concurrent-ruby (~> 1.0.5) connection_pool (~> 2.0) creole (~> 0.5.0) diff --git a/changelogs/unreleased/replace_redcarpet_with_cmark.yml b/changelogs/unreleased/replace_redcarpet_with_cmark.yml new file mode 100644 index 00000000000..7ce848b0bbd --- /dev/null +++ b/changelogs/unreleased/replace_redcarpet_with_cmark.yml @@ -0,0 +1,5 @@ +--- +title: Add CommonMark markdown engine (experimental) +merge_request: 14835 +author: blackst0ne +type: added diff --git a/config/initializers/8_metrics.rb b/config/initializers/8_metrics.rb index 45b39b2a38d..7cdf49159b4 100644 --- a/config/initializers/8_metrics.rb +++ b/config/initializers/8_metrics.rb @@ -94,6 +94,7 @@ def instrument_classes(instrumentation) instrumentation.instrument_instance_methods(RepositoryCheck::SingleRepositoryWorker) + instrumentation.instrument_instance_methods(Rouge::Plugins::CommonMark) instrumentation.instrument_instance_methods(Rouge::Plugins::Redcarpet) instrumentation.instrument_instance_methods(Rouge::Formatters::HTMLGitlab) diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb new file mode 100644 index 00000000000..bc9597df894 --- /dev/null +++ b/lib/banzai/filter/markdown_engines/common_mark.rb @@ -0,0 +1,45 @@ +# `CommonMark` markdown engine for GitLab's Banzai markdown filter. +# This module is used in Banzai::Filter::MarkdownFilter. +# Used gem is `commonmarker` which is a ruby wrapper for libcmark (CommonMark parser) +# including GitHub's GFM extensions. +# Homepage: https://github.com/gjtorikian/commonmarker + +module Banzai + module Filter + module MarkdownEngines + class CommonMark + EXTENSIONS = [ + :autolink, # provides support for automatically converting URLs to anchor tags. + :strikethrough, # provides support for strikethroughs. + :table, # provides support for tables. + :tagfilter # strips out several "unsafe" HTML tags from being used: https://github.github.com/gfm/#disallowed-raw-html-extension- + ].freeze + + PARSE_OPTIONS = [ + :FOOTNOTES, # parse footnotes. + :STRIKETHROUGH_DOUBLE_TILDE, # parse strikethroughs by double tildes (as redcarpet does). + :VALIDATE_UTF8 # replace illegal sequences with the replacement character U+FFFD. + ].freeze + + # The `:GITHUB_PRE_LANG` option is not used intentionally because + # it renders a fence block with language as `<pre lang="LANG"><code>some code\n</code></pre>` + # while GitLab's syntax is `<pre><code lang="LANG">some code\n</code></pre>`. + # If in the future the syntax is about to be made GitHub-compatible, please, add `:GITHUB_PRE_LANG` render option below + # and remove `code_block` method from `lib/banzai/renderer/common_mark/html.rb`. + RENDER_OPTIONS = [ + :DEFAULT # default rendering system. Nothing special. + ].freeze + + def initialize + @renderer = Banzai::Renderer::CommonMark::HTML.new(options: RENDER_OPTIONS) + end + + def render(text) + doc = CommonMarker.render_doc(text, PARSE_OPTIONS, EXTENSIONS) + + @renderer.render(doc) + end + end + end + end +end diff --git a/lib/banzai/filter/markdown_engines/redcarpet.rb b/lib/banzai/filter/markdown_engines/redcarpet.rb new file mode 100644 index 00000000000..ac99941fefa --- /dev/null +++ b/lib/banzai/filter/markdown_engines/redcarpet.rb @@ -0,0 +1,32 @@ +# `Redcarpet` markdown engine for GitLab's Banzai markdown filter. +# This module is used in Banzai::Filter::MarkdownFilter. +# Used gem is `redcarpet` which is a ruby library for markdown processing. +# Homepage: https://github.com/vmg/redcarpet + +module Banzai + module Filter + module MarkdownEngines + class Redcarpet + OPTIONS = { + fenced_code_blocks: true, + footnotes: true, + lax_spacing: true, + no_intra_emphasis: true, + space_after_headers: true, + strikethrough: true, + superscript: true, + tables: true + }.freeze + + def initialize + html_renderer = Banzai::Renderer::Redcarpet::HTML.new + @renderer = ::Redcarpet::Markdown.new(html_renderer, OPTIONS) + end + + def render(text) + @renderer.render(text) + end + end + end + end +end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index 9cac303e645..c1e2b680240 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -1,34 +1,31 @@ module Banzai module Filter class MarkdownFilter < HTML::Pipeline::TextFilter - # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use - REDCARPET_OPTIONS = { - fenced_code_blocks: true, - footnotes: true, - lax_spacing: true, - no_intra_emphasis: true, - space_after_headers: true, - strikethrough: true, - superscript: true, - tables: true - }.freeze - def initialize(text, context = nil, result = nil) - super text, context, result - @text = @text.delete "\r" + super(text, context, result) + + @renderer = renderer(context[:markdown_engine]).new + @text = @text.delete("\r") end def call - html = self.class.renderer.render(@text) - html.rstrip! - html + @renderer.render(@text).rstrip + end + + private + + DEFAULT_ENGINE = :redcarpet + + def engine(engine_from_context) + engine_from_context ||= DEFAULT_ENGINE + + engine_from_context.to_s.classify end - def self.renderer - Thread.current[:banzai_markdown_renderer] ||= begin - renderer = Banzai::Renderer::HTML.new - Redcarpet::Markdown.new(renderer, REDCARPET_OPTIONS) - end + def renderer(engine_from_context) + "Banzai::Filter::MarkdownEngines::#{engine(engine_from_context)}".constantize + rescue NameError + raise NameError, "`#{engine_from_context}` is unknown markdown engine" end end end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 0ac7e231b5b..6dbf0d68fe8 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,3 +1,4 @@ +require 'rouge/plugins/common_mark' require 'rouge/plugins/redcarpet' module Banzai diff --git a/lib/banzai/renderer/common_mark/html.rb b/lib/banzai/renderer/common_mark/html.rb new file mode 100644 index 00000000000..c7a54629f31 --- /dev/null +++ b/lib/banzai/renderer/common_mark/html.rb @@ -0,0 +1,21 @@ +module Banzai + module Renderer + module CommonMark + class HTML < CommonMarker::HtmlRenderer + def code_block(node) + block do + code = node.string_content + lang = node.fence_info + lang_attr = lang.present? ? %Q{ lang="#{lang}"} : '' + result = + "<pre>" \ + "<code#{lang_attr}>#{html_escape(code)}</code>" \ + "</pre>" + + out(result) + end + end + end + end + end +end diff --git a/lib/banzai/renderer/html.rb b/lib/banzai/renderer/html.rb deleted file mode 100644 index 252caa35947..00000000000 --- a/lib/banzai/renderer/html.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Banzai - module Renderer - class HTML < Redcarpet::Render::HTML - def block_code(code, lang) - lang_attr = lang ? %Q{ lang="#{lang}"} : '' - - "\n<pre>" \ - "<code#{lang_attr}>#{html_escape(code)}</code>" \ - "</pre>" - end - end - end -end diff --git a/lib/banzai/renderer/redcarpet/html.rb b/lib/banzai/renderer/redcarpet/html.rb new file mode 100644 index 00000000000..94df5d8b1e1 --- /dev/null +++ b/lib/banzai/renderer/redcarpet/html.rb @@ -0,0 +1,15 @@ +module Banzai + module Renderer + module Redcarpet + class HTML < ::Redcarpet::Render::HTML + def block_code(code, lang) + lang_attr = lang ? %Q{ lang="#{lang}"} : '' + + "\n<pre>" \ + "<code#{lang_attr}>#{html_escape(code)}</code>" \ + "</pre>" + end + end + end + end +end diff --git a/lib/rouge/plugins/common_mark.rb b/lib/rouge/plugins/common_mark.rb new file mode 100644 index 00000000000..8f9de061124 --- /dev/null +++ b/lib/rouge/plugins/common_mark.rb @@ -0,0 +1,20 @@ +# A rouge plugin for CommonMark markdown engine. +# Used to highlight code generated by CommonMark. + +module Rouge + module Plugins + module CommonMark + def code_block(code, language) + lexer = Lexer.find_fancy(language, code) || Lexers::PlainText + + formatter = rouge_formatter(lexer) + formatter.format(lexer.lex(code)) + end + + # override this method for custom formatting behavior + def rouge_formatter(lexer) + Formatters::HTMLLegacy.new(css_class: "highlight #{lexer.tag}") + end + end + end +end |