Merge branch 'issue_3945' into 'master'

Add syntax highlighting to diff view

Closes #3945 

See merge request !2109

9 files changed, 447 insertions, 121 deletions

diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index 79061cd0141..a484177ae33 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -1,13 +1,22 @@
 module Gitlab
   module Diff
     class File
-      attr_reader :diff
+      attr_reader :diff, :diff_refs
 
       delegate :new_file, :deleted_file, :renamed_file,
         :old_path, :new_path, to: :diff, prefix: false
 
-      def initialize(diff)
+      def initialize(diff, diff_refs)
         @diff = diff
+        @diff_refs = diff_refs
+      end
+
+      def old_ref
+        diff_refs[0] if diff_refs
+      end
+
+      def new_ref
+        diff_refs[1] if diff_refs
       end
 
       # Array of Gitlab::DIff::Line objects
@@ -15,6 +24,14 @@ module Gitlab
         @lines ||= parser.parse(raw_diff.lines)
       end
 
+      def highlighted_diff_lines
+        Gitlab::Diff::Highlight.new(self).highlight
+      end
+
+      def parallel_diff_lines
+        Gitlab::Diff::ParallelDiff.new(self).parallelize
+      end
+
       def mode_changed?
         !!(diff.a_mode && diff.b_mode && diff.a_mode != diff.b_mode)
       end
diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb
new file mode 100644
index 00000000000..179f8164c84
--- /dev/null
+++ b/lib/gitlab/diff/highlight.rb
@@ -0,0 +1,75 @@
+module Gitlab
+  module Diff
+    class Highlight
+      attr_reader :diff_file
+
+      delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff
+
+      def initialize(diff_file)
+        @diff_file = diff_file
+        @diff_lines = diff_file.diff_lines
+        @raw_lines = @diff_lines.map(&:text)
+      end
+
+      def highlight
+        @diff_lines.each_with_index do |diff_line, i|
+          # ignore highlighting for "match" lines
+          next if diff_line.type == 'match' || diff_line.type == 'nonewline'
+
+          rich_line = highlight_line(diff_line, i)
+
+          if line_inline_diffs = inline_diffs[i]
+            rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs)
+          end
+
+          diff_line.text = rich_line.html_safe
+        end
+
+        @diff_lines
+      end
+
+      private
+
+      def highlight_line(diff_line, index)
+        return html_escape(diff_line.text) unless diff_file.diff_refs
+
+        line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' '
+
+        case diff_line.type
+        when 'new', nil
+          rich_line = new_lines[diff_line.new_pos - 1]
+        when 'old'
+          rich_line = old_lines[diff_line.old_pos - 1]
+        end
+
+        # Only update text if line is found. This will prevent
+        # issues with submodules given the line only exists in diff content.
+        rich_line ? line_prefix + rich_line : html_escape(diff_line.text)
+      end
+
+      def inline_diffs
+        @inline_diffs ||= InlineDiff.new(@raw_lines).inline_diffs
+      end
+
+      def old_lines
+        @old_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:old))
+      end
+
+      def new_lines
+        @new_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:new))
+      end
+
+      def processing_args(version)
+        ref  = send("diff_#{version}_ref")
+        path = send("diff_#{version}_path")
+
+        [ref.project.repository, ref.id, path]
+      end
+
+      def html_escape(str)
+        replacements = { '&' => '&amp;', '>' => '&gt;', '<' => '&lt;', '"' => '&quot;', "'" => '&#39;' }
+        str.gsub(/[&"'><]/, replacements)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb
new file mode 100644
index 00000000000..b8a61ad6115
--- /dev/null
+++ b/lib/gitlab/diff/inline_diff.rb
@@ -0,0 +1,73 @@
+module Gitlab
+  module Diff
+    class InlineDiff
+      attr_accessor :lines
+
+      def initialize(lines)
+        @lines = lines
+      end
+
+      def inline_diffs
+        inline_diffs = []
+
+        local_edit_indexes.each do |index|
+          old_index = index
+          new_index = index + 1
+          old_line = @lines[old_index]
+          new_line = @lines[new_index]
+
+          # Skip inline diff if empty line was replaced with content
+          next if old_line[1..-1] == ""
+
+          # Add one, because this is based on the prefixless version
+          lcp = longest_common_prefix(old_line[1..-1], new_line[1..-1]) + 1
+          lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1])
+
+          old_diff_range = lcp..(old_line.length - lcs - 1)
+          new_diff_range = lcp..(new_line.length - lcs - 1)
+
+          inline_diffs[old_index] = [old_diff_range] if old_diff_range.begin <= old_diff_range.end
+          inline_diffs[new_index] = [new_diff_range] if new_diff_range.begin <= new_diff_range.end
+        end
+
+        inline_diffs
+      end
+
+      private
+
+      # Find runs of single line edits
+      def local_edit_indexes
+        line_prefixes = @lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' }
+        joined_line_prefixes = " #{line_prefixes.join} "
+
+        offset = 0
+        local_edit_indexes = []
+        while index = joined_line_prefixes.index(" -+ ", offset)
+          local_edit_indexes << index
+          offset = index + 1
+        end
+
+        local_edit_indexes
+      end
+
+      def longest_common_prefix(a, b)
+        max_length = [a.length, b.length].max
+
+        length = 0
+        (0..max_length - 1).each do |pos|
+          old_char = a[pos]
+          new_char = b[pos]
+
+          break if old_char != new_char
+          length += 1
+        end
+
+        length
+      end
+
+      def longest_common_suffix(a, b)
+        longest_common_prefix(a.reverse, b.reverse)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb
new file mode 100644
index 00000000000..1d7fa1bce06
--- /dev/null
+++ b/lib/gitlab/diff/inline_diff_marker.rb
@@ -0,0 +1,109 @@
+module Gitlab
+  module Diff
+    class InlineDiffMarker
+      attr_accessor :raw_line, :rich_line
+
+      def initialize(raw_line, rich_line = raw_line)
+        @raw_line = raw_line
+        @rich_line = rich_line
+      end
+
+      def mark(line_inline_diffs)
+        marker_ranges = []
+        line_inline_diffs.each do |inline_diff_range|
+          # Map the inline-diff range based on the raw line to character positions in the rich line
+          inline_diff_positions = position_mapping[inline_diff_range].flatten
+          # Turn the array of character positions into ranges
+          marker_ranges.concat(collapse_ranges(inline_diff_positions))
+        end
+
+        offset = 0
+        # Mark each range
+        marker_ranges.each do |range|
+          offset = insert_around_range(rich_line, range, "<span class='idiff'>", "</span>", offset)
+        end
+
+        rich_line
+      end
+
+      private
+
+      # Mapping of character positions in the raw line, to the rich (highlighted) line
+      def position_mapping
+        @position_mapping ||= begin
+          mapping = []
+          rich_pos = 0
+          (0..raw_line.length).each do |raw_pos|
+            rich_char = rich_line[rich_pos]
+
+            # The raw and rich lines are the same except for HTML tags,
+            # so skip over any `<...>` segment
+            while rich_char == '<'
+              until rich_char == '>'
+                rich_pos += 1
+                rich_char = rich_line[rich_pos]
+              end
+
+              rich_pos += 1
+              rich_char = rich_line[rich_pos]
+            end
+
+            # multi-char HTML entities in the rich line correspond to a single character in the raw line
+            if rich_char == '&'
+              multichar_mapping = [rich_pos]
+              until rich_char == ';'
+                rich_pos += 1
+                multichar_mapping << rich_pos
+                rich_char = rich_line[rich_pos]
+              end
+
+              mapping[raw_pos] = multichar_mapping
+            else
+              mapping[raw_pos] = rich_pos
+            end
+
+            rich_pos += 1
+          end
+
+          mapping
+        end
+      end
+
+      # Takes an array of integers, and returns an array of ranges covering the same integers
+      def collapse_ranges(positions)
+        return [] if positions.empty?
+        ranges = []
+
+        start = prev = positions[0]
+        range = start..prev
+        positions[1..-1].each do |pos|
+          if pos == prev + 1
+            range = start..pos
+            prev = pos
+          else
+            ranges << range
+            start = prev = pos
+            range = start..prev
+          end
+        end
+        ranges << range
+
+        ranges
+      end
+
+      # Inserts tags around the characters identified by the given range
+      def insert_around_range(text, range, before, after, offset = 0)
+        # Just to be sure
+        return offset if offset + range.end + 1 > text.length
+
+        text.insert(offset + range.begin, before)
+        offset += before.length
+
+        text.insert(offset + range.end + 1, after)
+        offset += after.length
+
+        offset
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index 0072194606e..03730b435ad 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -1,7 +1,8 @@
 module Gitlab
   module Diff
     class Line
-      attr_reader :type, :text, :index, :old_pos, :new_pos
+      attr_reader :type, :index, :old_pos, :new_pos
+      attr_accessor :text
 
       def initialize(text, type, index, old_pos, new_pos)
         @text, @type, @index = text, type, index
diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb
new file mode 100644
index 00000000000..c0db3559e3a
--- /dev/null
+++ b/lib/gitlab/diff/parallel_diff.rb
@@ -0,0 +1,117 @@
+module Gitlab
+  module Diff
+    class ParallelDiff
+      attr_accessor :diff_file
+
+      def initialize(diff_file)
+        @diff_file = diff_file
+      end
+
+      def parallelize
+        lines = []
+        skip_next = false
+
+        diff_file.highlighted_diff_lines.each do |line|
+          full_line = line.text
+          type = line.type
+          line_code = generate_line_code(diff_file.file_path, line)
+          line_new = line.new_pos
+          line_old = line.old_pos
+
+          next_line = diff_file.next_line(line.index)
+
+          if next_line
+            next_line_code = generate_line_code(diff_file.file_path, next_line)
+            next_type = next_line.type
+            next_line = next_line.text
+          end
+
+          case type
+          when 'match', nil
+            # line in the right panel is the same as in the left one
+            lines << {
+              left: {
+                type:       type,
+                number:     line_old,
+                text:       full_line,
+                line_code:  line_code,
+              },
+              right: {
+                type:       type,
+                number:     line_new,
+                text:       full_line,
+                line_code:  line_code
+              }
+            }
+          when 'old'
+            case next_type
+            when 'new'
+              # Left side has text removed, right side has text added
+              lines << {
+                left: {
+                  type:       type,
+                  number:     line_old,
+                  text:       full_line,
+                  line_code:  line_code,
+                },
+                right: {
+                  type:       next_type,
+                  number:     line_new,
+                  text:       next_line,
+                  line_code:  next_line_code
+                }
+              }
+              skip_next = true
+            when 'old', 'nonewline', nil
+              # Left side has text removed, right side doesn't have any change
+              # No next line code, no new line number, no new line text
+              lines << {
+                left: {
+                  type:       type,
+                  number:     line_old,
+                  text:       full_line,
+                  line_code:  line_code,
+                },
+                right: {
+                  type:       next_type,
+                  number:     nil,
+                  text:       "",
+                  line_code:  nil
+                }
+              }
+            end
+          when 'new'
+            if skip_next
+              # Change has been already included in previous line so no need to do it again
+              skip_next = false
+              next
+            else
+              # Change is only on the right side, left side has no change
+              lines << {
+                left: {
+                  type:       nil,
+                  number:     nil,
+                  text:       "",
+                  line_code:  line_code,
+                },
+                right: {
+                  type:       type,
+                  number:     line_new,
+                  text:       full_line,
+                  line_code:  line_code
+                }
+              }
+            end
+          end
+        end
+        lines
+      end
+
+      private
+
+      def generate_line_code(file_path, line)
+        Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb
index 516e59b87a3..3666063bf8b 100644
--- a/lib/gitlab/diff/parser.rb
+++ b/lib/gitlab/diff/parser.rb
@@ -11,13 +11,10 @@ module Gitlab
         line_new = 1
         type = nil
 
-        lines_arr = ::Gitlab::InlineDiff.processing lines
-
-        lines_arr.each do |line|
+        @lines.each do |line|
           next if filename?(line)
 
-          full_line = html_escape(line.gsub(/\n/, ''))
-          full_line = ::Gitlab::InlineDiff.replace_markers full_line
+          full_line = line.gsub(/\n/, '')
 
           if line.match(/^@@ -/)
             type = "match"
@@ -29,6 +26,10 @@ module Gitlab
             lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new)
             line_obj_index += 1
             next
+          elsif line[0] == '\\'
+            type = 'nonewline'
+            lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new)
+            line_obj_index += 1
           else
             type = identification_type(line)
             lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new)
@@ -36,10 +37,13 @@ module Gitlab
           end
 
 
-          if line[0] == "+"
+          case line[0]
+          when "+"
             line_new += 1
-          elsif line[0] == "-"
+          when "-"
             line_old += 1
+          when "\\"
+            # No increment
           else
             line_new += 1
             line_old += 1
@@ -62,19 +66,15 @@ module Gitlab
       end
 
       def identification_type(line)
-        if line[0] == "+"
+        case line[0]
+        when "+"
           "new"
-        elsif line[0] == "-"
+        when "-"
           "old"
         else
           nil
         end
       end
-
-      def html_escape(str)
-        replacements = { '&' => '&amp;', '>' => '&gt;', '<' => '&lt;', '"' => '&quot;', "'" => '&#39;' }
-        str.gsub(/[&"'><]/, replacements)
-      end
     end
   end
 end
diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb
new file mode 100644
index 00000000000..4ddb4fea977
--- /dev/null
+++ b/lib/gitlab/highlight.rb
@@ -0,0 +1,38 @@
+module Gitlab
+  class Highlight
+    def self.highlight(blob_name, blob_content, nowrap: true)
+      new(blob_name, blob_content, nowrap: nowrap).highlight(blob_content, continue: false)
+    end
+
+    def self.highlight_lines(repository, ref, file_name)
+      blob = repository.blob_at(ref, file_name)
+      return [] unless blob
+
+      highlight(file_name, blob.data).lines.map!(&:html_safe)
+    end
+
+    def initialize(blob_name, blob_content, nowrap: true)
+      @formatter = rouge_formatter(nowrap: nowrap)
+      @lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText
+    end
+
+    def highlight(text, continue: true)
+      @formatter.format(@lexer.lex(text, continue: continue)).html_safe
+    rescue
+      @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe
+    end
+
+    private
+
+    def rouge_formatter(options = {})
+      options = options.reverse_merge(
+        nowrap: true,
+        cssclass: 'code highlight',
+        lineanchors: true,
+        lineanchorsid: 'LC'
+      )
+
+      Rouge::Formatters::HTMLGitlab.new(options)
+    end
+  end
+end
diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb
deleted file mode 100644
index 44507bde25d..00000000000
--- a/lib/gitlab/inline_diff.rb
+++ /dev/null
@@ -1,104 +0,0 @@
-module Gitlab
-  class InlineDiff
-    class << self
-
-      START  = "#!idiff-start!#"
-      FINISH = "#!idiff-finish!#"
-
-      def processing(diff_arr)
-        indexes = _indexes_of_changed_lines diff_arr
-
-        indexes.each do |index|
-          first_line = diff_arr[index+1]
-          second_line = diff_arr[index+2]
-
-          # Skip inline diff if empty line was replaced with content
-          next if first_line == "-\n"
-
-          first_token = find_first_token(first_line, second_line)
-          apply_first_token(diff_arr, index, first_token)
-
-          last_token = find_last_token(first_line, second_line, first_token)
-          apply_last_token(diff_arr, index, last_token)
-        end
-
-        diff_arr
-      end
-
-      def apply_first_token(diff_arr, index, first_token)
-        start = first_token + START
-
-        if first_token.empty?
-          # In case if we remove string of spaces in commit
-          diff_arr[index+1].sub!("-", "-" => "-#{START}")
-          diff_arr[index+2].sub!("+", "+" => "+#{START}")
-        else
-          diff_arr[index+1].sub!(first_token, first_token => start)
-          diff_arr[index+2].sub!(first_token, first_token => start)
-        end
-      end
-
-      def apply_last_token(diff_arr, index, last_token)
-        # This is tricky: escape backslashes so that `sub` doesn't interpret them
-        # as backreferences. Regexp.escape does NOT do the right thing.
-        replace_token = FINISH + last_token.gsub(/\\/, '\&\&')
-        diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token)
-        diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token)
-      end
-
-      def find_first_token(first_line, second_line)
-        max_length = [first_line.size, second_line.size].max
-        first_the_same_symbols = 0
-
-        (0..max_length + 1).each do |i|
-          first_the_same_symbols = i - 1
-
-          if first_line[i] != second_line[i] && i > 0
-            break
-          end
-        end
-
-        first_line[0..first_the_same_symbols][1..-1]
-      end
-
-      def find_last_token(first_line, second_line, first_token)
-        max_length = [first_line.size, second_line.size].max
-        last_the_same_symbols = 0
-
-        (1..max_length + 1).each do |i|
-          last_the_same_symbols = -i
-          shortest_line = second_line.size > first_line.size ? first_line : second_line
-
-          if (first_line[-i] != second_line[-i]) || "#{first_token}#{START}".size == shortest_line[1..-i].size
-            break
-          end
-        end
-
-        last_the_same_symbols += 1
-        first_line[last_the_same_symbols..-1]
-      end
-
-      def _indexes_of_changed_lines(diff_arr)
-        chain_of_first_symbols = ""
-        diff_arr.each_with_index do |line, i|
-          chain_of_first_symbols += line[0]
-        end
-        chain_of_first_symbols.gsub!(/[^\-\+]/, "#")
-
-        offset = 0
-        indexes = []
-        while index = chain_of_first_symbols.index("#-+#", offset)
-          indexes << index
-          offset = index + 1
-        end
-        indexes
-      end
-
-      def replace_markers(line)
-        line.gsub!(START, "<span class='idiff'>")
-        line.gsub!(FINISH, "</span>")
-        line
-      end
-    end
-  end
-end