Merge branch 'tmp-reference-pipeline-and-caching' into 'master'

[Second try] Implement different Markdown rendering pipelines and cache Markdown

!1602 already got merged in bcd89a58e736685bcce662fe0acf793ee925b536, but it would appear the merge commit disappeared because of #3816 (or some other reason).

cc @rspeicher 

See merge request !2051

21 files changed, 68 insertions, 72 deletions

diff --git a/app/helpers/gitlab_markdown_helper.rb b/app/helpers/gitlab_markdown_helper.rb
index 98c6d9d5d2e..5004e02ea0b 100644
--- a/app/helpers/gitlab_markdown_helper.rb
+++ b/app/helpers/gitlab_markdown_helper.rb
@@ -20,7 +20,7 @@ module GitlabMarkdownHelper
                    end
 
     user = current_user if defined?(current_user)
-    gfm_body = Gitlab::Markdown.gfm(escaped_body, project: @project, current_user: user)
+    gfm_body = Gitlab::Markdown.render(escaped_body, project: @project, current_user: user, pipeline: :single_line)
 
     fragment = Nokogiri::HTML::DocumentFragment.parse(gfm_body)
     if fragment.children.size == 1 && fragment.children[0].name == 'a'
@@ -46,23 +46,35 @@ module GitlabMarkdownHelper
   end
 
   def markdown(text, context = {})
-    process_markdown(text, context)
-  end
+    return "" unless text.present?
+
+    context[:project] ||= @project
 
-  # TODO (rspeicher): Remove all usages of this helper and just call `markdown`
-  # with a custom pipeline depending on the content being rendered
-  def gfm(text, options = {})
-    process_markdown(text, options, :gfm)
+    html = Gitlab::Markdown.render(text, context)
+
+    context.merge!(
+      current_user:   (current_user if defined?(current_user)),
+
+      # RelativeLinkFilter
+      requested_path: @path,
+      project_wiki:   @project_wiki,
+      ref:            @ref
+    )
+
+    Gitlab::Markdown.post_process(html, context)
   end
 
   def asciidoc(text)
-    Gitlab::Asciidoc.render(text, {
-      commit: @commit,
-      project: @project,
-      project_wiki: @project_wiki,
+    Gitlab::Asciidoc.render(text,
+      project:      @project,
+      current_user: (current_user if defined?(current_user)),
+
+      # RelativeLinkFilter
+      project_wiki:   @project_wiki,
       requested_path: @path,
-      ref: @ref
-    })
+      ref:            @ref,
+      commit:         @commit
+    )
   end
 
   # Return the first line of +text+, up to +max_chars+, after parsing the line
@@ -178,26 +190,4 @@ module GitlabMarkdownHelper
       ''
     end
   end
-
-  def process_markdown(text, options, method = :markdown)
-    return "" unless text.present?
-
-    options.reverse_merge!(
-      path:         @path,
-      pipeline:     :default,
-      project:      @project,
-      project_wiki: @project_wiki,
-      ref:          @ref
-    )
-
-    user = current_user if defined?(current_user)
-
-    html = if method == :gfm
-             Gitlab::Markdown.gfm(text, options)
-           else
-             Gitlab::Markdown.render(text, options)
-           end
-
-    Gitlab::Markdown.post_process(html, pipeline: options[:pipeline], project: @project, user: user)
-  end
 end
diff --git a/app/models/commit.rb b/app/models/commit.rb
index fa88a408fa3..0ba7b584d91 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -7,7 +7,7 @@ class Commit
   include Referable
   include StaticModel
 
-  attr_mentionable :safe_message
+  attr_mentionable :safe_message, pipeline: :single_line
   participant :author, :committer, :notes
 
   attr_accessor :project
diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb
index badeadfa418..f56fd3e02d4 100644
--- a/app/models/concerns/issuable.rb
+++ b/app/models/concerns/issuable.rb
@@ -50,7 +50,8 @@ module Issuable
              allow_nil: true,
              prefix: true
 
-    attr_mentionable :title, :description
+    attr_mentionable :title, pipeline: :single_line
+    attr_mentionable :description, cache: true
     participant :author, :assignee, :notes_with_associations
     strip_attributes :title
   end
diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb
index 634a8d0f274..d2ea9ab7313 100644
--- a/app/models/concerns/mentionable.rb
+++ b/app/models/concerns/mentionable.rb
@@ -10,8 +10,9 @@ module Mentionable
 
   module ClassMethods
     # Indicate which attributes of the Mentionable to search for GFM references.
-    def attr_mentionable(*attrs)
-      mentionable_attrs.concat(attrs.map(&:to_s))
+    def attr_mentionable(attr, options = {})
+      attr = attr.to_s
+      mentionable_attrs << [attr, options]
     end
 
     # Accessor for attributes marked mentionable.
@@ -37,19 +38,24 @@ module Mentionable
     "#{friendly_name} #{to_reference(from_project)}"
   end
 
-  # Construct a String that contains possible GFM references.
-  def mentionable_text
-    self.class.mentionable_attrs.map { |attr| send(attr) }.compact.join("\n\n")
-  end
-
   # The GFM reference to this Mentionable, which shouldn't be included in its #references.
   def local_reference
     self
   end
 
-  def all_references(current_user = self.author, text = self.mentionable_text, load_lazy_references: true)
+  def all_references(current_user = self.author, text = nil, load_lazy_references: true)
     ext = Gitlab::ReferenceExtractor.new(self.project, current_user, load_lazy_references: load_lazy_references)
-    ext.analyze(text)
+    
+    if text
+      ext.analyze(text)
+    else
+      self.class.mentionable_attrs.each do |attr, options|
+        text = send(attr)
+        options[:cache_key] = [self, attr] if options.delete(:cache)
+        ext.analyze(text, options)
+      end
+    end
+
     ext
   end
 
@@ -58,9 +64,7 @@ module Mentionable
   end
 
   # Extract GFM references to other Mentionables from this Mentionable. Always excludes its #local_reference.
-  def referenced_mentionables(current_user = self.author, text = self.mentionable_text, load_lazy_references: true)
-    return [] if text.blank?
-
+  def referenced_mentionables(current_user = self.author, text = nil, load_lazy_references: true)
     refs = all_references(current_user, text, load_lazy_references: load_lazy_references)
     refs = (refs.issues + refs.merge_requests + refs.commits)
 
@@ -70,8 +74,8 @@ module Mentionable
     refs.reject { |ref| ref == local_reference }
   end
 
-  # Create a cross-reference Note for each GFM reference to another Mentionable found in +mentionable_text+.
-  def create_cross_references!(author = self.author, without = [], text = self.mentionable_text)
+  # Create a cross-reference Note for each GFM reference to another Mentionable found in the +mentionable_attrs+.
+  def create_cross_references!(author = self.author, without = [], text = nil)
     refs = referenced_mentionables(author, text)
 
     # We're using this method instead of Array diffing because that requires
@@ -111,7 +115,7 @@ module Mentionable
   def detect_mentionable_changes
     source = (changes.present? ? changes : previous_changes).dup
 
-    mentionable = self.class.mentionable_attrs
+    mentionable = self.class.mentionable_attrs.map { |attr, options| attr }
 
     # Only include changed fields that are mentionable
     source.select { |key, val| mentionable.include?(key) }
diff --git a/app/models/note.rb b/app/models/note.rb
index 98c29ddc4cd..de9392adbf4 100644
--- a/app/models/note.rb
+++ b/app/models/note.rb
@@ -29,7 +29,7 @@ class Note < ActiveRecord::Base
 
   default_value_for :system, false
 
-  attr_mentionable :note
+  attr_mentionable :note, cache: true, pipeline: :note
   participant :author
 
   belongs_to :project
diff --git a/app/models/project.rb b/app/models/project.rb
index cb965ce1b9e..e78868af1cc 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -662,6 +662,7 @@ class Project < ActiveRecord::Base
         gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
         send_move_instructions(old_path_with_namespace)
         reset_events_cache
+        @repository = nil
       rescue
         # Returning false does not rollback after_* transaction but gives
         # us information about failing some of tasks
diff --git a/app/views/dashboard/milestones/show.html.haml b/app/views/dashboard/milestones/show.html.haml
index 3536bbeaf4b..44b7efe5232 100644
--- a/app/views/dashboard/milestones/show.html.haml
+++ b/app/views/dashboard/milestones/show.html.haml
@@ -12,7 +12,7 @@
 
   .gray-content-block.middle-block
     %h2.issue-title
-      = gfm escape_once(@milestone.title)
+      = markdown escape_once(@milestone.title), pipeline: :single_line
 
 - if @milestone.complete? && @milestone.active?
   .alert.alert-success.prepend-top-default
diff --git a/app/views/events/_commit.html.haml b/app/views/events/_commit.html.haml
index ad63841ccf3..4ba8b84fd92 100644
--- a/app/views/events/_commit.html.haml
+++ b/app/views/events/_commit.html.haml
@@ -2,4 +2,4 @@
   .commit-row-title
     = link_to truncate_sha(commit[:id]), namespace_project_commit_path(project.namespace, project, commit[:id]), class: "commit_short_id", alt: ''
     &middot;
-    = gfm event_commit_title(commit[:message]), project: project
+    = markdown event_commit_title(commit[:message]), project: project, pipeline: :single_line
diff --git a/app/views/groups/milestones/show.html.haml b/app/views/groups/milestones/show.html.haml
index 3c1d8815013..350e216fcc6 100644
--- a/app/views/groups/milestones/show.html.haml
+++ b/app/views/groups/milestones/show.html.haml
@@ -18,7 +18,7 @@
 
   .gray-content-block.middle-block
     %h2.issue-title
-      = gfm escape_once(@milestone.title)
+      = markdown escape_once(@milestone.title), pipeline: :single_line
 
 - if @milestone.complete? && @milestone.active?
   .alert.alert-success.prepend-top-default
diff --git a/app/views/projects/commit/_commit_box.html.haml b/app/views/projects/commit/_commit_box.html.haml
index bb37e4a7049..132cdc35c94 100644
--- a/app/views/projects/commit/_commit_box.html.haml
+++ b/app/views/projects/commit/_commit_box.html.haml
@@ -52,10 +52,10 @@
 
 .commit-box.gray-content-block.middle-block
   %h3.commit-title
-    = gfm escape_once(@commit.title)
+    = markdown escape_once(@commit.title), pipeline: :single_line
   - if @commit.description.present?
     %pre.commit-description
-      = preserve(gfm(escape_once(@commit.description)))
+      = preserve(markdown(escape_once(@commit.description), pipeline: :single_line))
 
 :javascript
   $(".commit-info-row.branches").load("#{branches_namespace_project_commit_path(@project.namespace, @project, @commit.id)}");
diff --git a/app/views/projects/commits/_commit.html.haml b/app/views/projects/commits/_commit.html.haml
index 2e489a0a4d5..0d64486164e 100644
--- a/app/views/projects/commits/_commit.html.haml
+++ b/app/views/projects/commits/_commit.html.haml
@@ -32,7 +32,7 @@
     - if commit.description?
       .commit-row-description.js-toggle-content
         %pre
-          = preserve(gfm(escape_once(commit.description)))
+          = preserve(markdown(escape_once(commit.description), pipeline: :single_line))
 
     .commit-row-info
       = commit_author_link(commit, avatar: true, size: 24)
diff --git a/app/views/projects/commits/show.atom.builder b/app/views/projects/commits/show.atom.builder
index 268b9b815ee..7ffa7317196 100644
--- a/app/views/projects/commits/show.atom.builder
+++ b/app/views/projects/commits/show.atom.builder
@@ -17,7 +17,7 @@ xml.feed "xmlns" => "http://www.w3.org/2005/Atom", "xmlns:media" => "http://sear
         xml.name commit.author_name
         xml.email commit.author_email
       end
-      xml.summary gfm(commit.description)
+      xml.summary markdown(commit.description, pipeline: :single_line)
     end
   end
 end
diff --git a/app/views/projects/edit.html.haml b/app/views/projects/edit.html.haml
index b28ada909b7..d865d299135 100644
--- a/app/views/projects/edit.html.haml
+++ b/app/views/projects/edit.html.haml
@@ -24,10 +24,10 @@
               .col-sm-10
                 = f.text_area :description, class: "form-control", rows: 3, maxlength: 250
 
-            - if @project.repository.exists? && @project.repository.branch_names.any?
+            - unless @project.empty_repo?
               .form-group
                 = f.label :default_branch, "Default Branch", class: 'control-label'
-                .col-sm-10= f.select(:default_branch, @repository.branch_names, {}, {class: 'select2 select-wide'})
+                .col-sm-10= f.select(:default_branch, @project.repository.branch_names, {}, {class: 'select2 select-wide'})
 
 
           = render 'shared/visibility_level', f: f, visibility_level: @project.visibility_level, can_change_visibility_level: can_change_visibility_level?(@project, current_user), form_model: @project
diff --git a/app/views/projects/issues/_closed_by_box.html.haml b/app/views/projects/issues/_closed_by_box.html.haml
index 917d5181689..3c491c1a8b8 100644
--- a/app/views/projects/issues/_closed_by_box.html.haml
+++ b/app/views/projects/issues/_closed_by_box.html.haml
@@ -1,3 +1,3 @@
 .issue-closed-by-widget
   = icon('check')
-  This issue will be closed automatically when merge request #{gfm(merge_requests_sentence(@closed_by_merge_requests))} is accepted.
+  This issue will be closed automatically when merge request #{markdown(merge_requests_sentence(@closed_by_merge_requests), pipeline: :gfm)} is accepted.
diff --git a/app/views/projects/issues/show.html.haml b/app/views/projects/issues/show.html.haml
index e2de17cee6d..a78d20cc07e 100644
--- a/app/views/projects/issues/show.html.haml
+++ b/app/views/projects/issues/show.html.haml
@@ -38,13 +38,13 @@
 
     .gray-content-block.middle-block
       %h2.issue-title
-        = gfm escape_once(@issue.title)
+        = markdown escape_once(@issue.title), pipeline: :single_line
       %div
         - if @issue.description.present?
           .description{class: can?(current_user, :update_issue, @issue) ? 'js-task-list-container' : ''}
             .wiki
               = preserve do
-                = markdown(@issue.description)
+                = markdown(@issue.description, cache_key: [@issue, "description"])
             %textarea.hidden.js-task-list-field
               = @issue.description
   - if @closed_by_merge_requests.present?
diff --git a/app/views/projects/merge_requests/show/_mr_box.html.haml b/app/views/projects/merge_requests/show/_mr_box.html.haml
index b4f62a75890..9bfe202589e 100644
--- a/app/views/projects/merge_requests/show/_mr_box.html.haml
+++ b/app/views/projects/merge_requests/show/_mr_box.html.haml
@@ -1,12 +1,12 @@
 .gray-content-block.middle-block
   %h2.issue-title
-    = gfm escape_once(@merge_request.title)
+    = markdown escape_once(@merge_request.title), pipeline: :single_line
 
   %div
     - if @merge_request.description.present?
       .description{class: can?(current_user, :update_merge_request, @merge_request) ? 'js-task-list-container' : ''}
         .wiki
           = preserve do
-            = markdown(@merge_request.description)
+            = markdown(@merge_request.description, cache_key: [@merge_request, "description"])
         %textarea.hidden.js-task-list-field
           = @merge_request.description
diff --git a/app/views/projects/merge_requests/widget/_open.html.haml b/app/views/projects/merge_requests/widget/_open.html.haml
index e0013fb769a..55dbae598d3 100644
--- a/app/views/projects/merge_requests/widget/_open.html.haml
+++ b/app/views/projects/merge_requests/widget/_open.html.haml
@@ -26,4 +26,4 @@
         %i.fa.fa-check
         Accepting this merge request will close #{"issue".pluralize(@closes_issues.size)}
         = succeed '.' do
-          != gfm(issues_sentence(@closes_issues))
+          != markdown issues_sentence(@closes_issues), pipeline: :gfm
diff --git a/app/views/projects/milestones/show.html.haml b/app/views/projects/milestones/show.html.haml
index c3bda794c65..7ecee440337 100644
--- a/app/views/projects/milestones/show.html.haml
+++ b/app/views/projects/milestones/show.html.haml
@@ -32,7 +32,7 @@
 
   .gray-content-block.middle-block
     %h2.issue-title
-      = gfm escape_once(@milestone.title)
+      = markdown escape_once(@milestone.title), pipeline: :single_line
     %div
       - if @milestone.description.present?
         .description
diff --git a/app/views/projects/notes/_note.html.haml b/app/views/projects/notes/_note.html.haml
index dd0abc8c746..922535e5c4a 100644
--- a/app/views/projects/notes/_note.html.haml
+++ b/app/views/projects/notes/_note.html.haml
@@ -38,7 +38,7 @@
       .note-body{class: note_editable?(note) ? 'js-task-list-container' : ''}
         .note-text
           = preserve do
-            = markdown(note.note, {no_header_anchors: true})
+            = markdown(note.note, pipeline: :note, cache_key: [note, "note"])
         - if note_editable?(note)
           = render 'projects/notes/edit_form', note: note
 
diff --git a/app/views/projects/repositories/_feed.html.haml b/app/views/projects/repositories/_feed.html.haml
index f3526ad0747..6ca919f7f80 100644
--- a/app/views/projects/repositories/_feed.html.haml
+++ b/app/views/projects/repositories/_feed.html.haml
@@ -12,7 +12,7 @@
       = link_to namespace_project_commits_path(@project.namespace, @project, commit.id) do
         %code= commit.short_id
       = image_tag avatar_icon(commit.author_email), class: "", width: 16, alt: ''
-      = gfm escape_once(truncate(commit.title, length: 40))
+      = markdown escape_once(truncate(commit.title, length: 40)), pipeline: :single_line
   %td
     %span.pull-right.cgray
       = time_ago_with_tooltip(commit.committed_date)
diff --git a/app/views/shared/snippets/_header.html.haml b/app/views/shared/snippets/_header.html.haml
index eb0fd21c2d4..669e6119fb6 100644
--- a/app/views/shared/snippets/_header.html.haml
+++ b/app/views/shared/snippets/_header.html.haml
@@ -22,4 +22,4 @@
 
   .gray-content-block.middle-block
     %h2.issue-title
-      = gfm escape_once(@snippet.title)
+      = markdown escape_once(@snippet.title), pipeline: :single_line