Add latest changes from gitlab-org/gitlab@master

6 files changed, 168 insertions, 19 deletions

diff --git a/lib/gitlab/diff/custom_diff.rb b/lib/gitlab/diff/custom_diff.rb
index 3928ece9281..5fb636fc4e0 100644
--- a/lib/gitlab/diff/custom_diff.rb
+++ b/lib/gitlab/diff/custom_diff.rb
@@ -16,10 +16,12 @@ module Gitlab
         end
 
         def transformed_diff(before, after)
+          Gitlab::AppLogger.info({ message: 'IPYNB_DIFF_GENERATED' })
+
           transformed_diff = IpynbDiff.diff(before, after,
-                                            diff_opts: { context: 5, include_diff_info: true },
-                                            transform_options: { cell_decorator: :percent },
-                                            raise_if_invalid_notebook: true)
+                         raise_if_invalid_nb: true,
+                         diffy_opts: { include_diff_info: true }).to_s(:text)
+
           strip_diff_frontmatter(transformed_diff)
         end
 
@@ -29,9 +31,7 @@ module Gitlab
 
         def transformed_blob_data(blob)
           if transformed_for_diff?(blob)
-            IpynbDiff.transform(blob.data,
-                                raise_errors: true,
-                                options: { include_metadata: false, cell_decorator: :percent })
+            IpynbDiff.transform(blob.data, raise_errors: true, include_frontmatter: false)
           end
         end
 
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index d9860d9fb86..6fc6c8bb36c 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -44,11 +44,15 @@ module Gitlab
         new_blob_lazy
         old_blob_lazy
 
-        diff.diff = Gitlab::Diff::CustomDiff.preprocess_before_diff(diff.new_path, old_blob_lazy, new_blob_lazy) || diff.diff if use_custom_diff?
+        diff.diff = Gitlab::Diff::CustomDiff.preprocess_before_diff(diff.new_path, old_blob_lazy, new_blob_lazy) || diff.diff unless use_renderable_diff?
       end
 
-      def use_custom_diff?
-        strong_memoize(:_custom_diff_enabled) { Feature.enabled?(:jupyter_clean_diffs, repository.project, default_enabled: true) }
+      def use_renderable_diff?
+        strong_memoize(:_renderable_diff_enabled) { Feature.enabled?(:rendered_diffs_viewer, repository.project, default_enabled: :yaml) }
+      end
+
+      def has_renderable?
+        rendered&.has_renderable?
       end
 
       def position(position_marker, position_type: :text)
@@ -370,6 +374,12 @@ module Gitlab
         lines.none? { |line| line.type.to_s == 'match' }
       end
 
+      def rendered
+        return unless use_renderable_diff? && ipynb?
+
+        strong_memoize(:rendered) { Rendered::Notebook::DiffFile.new(self) }
+      end
+
       private
 
       def diffable_by_attribute?
@@ -399,6 +409,10 @@ module Gitlab
         new_file? || deleted_file? || content_changed?
       end
 
+      def ipynb?
+        modified_file? && file_path.ends_with?('.ipynb')
+      end
+
       # We can't use Object#try because Blob doesn't inherit from Object, but
       # from BasicObject (via SimpleDelegator).
       def try_blobs(meth)
diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb
index f73e060be7f..7fa1bd6b5ec 100644
--- a/lib/gitlab/diff/file_collection/base.rb
+++ b/lib/gitlab/diff/file_collection/base.rb
@@ -11,7 +11,7 @@ module Gitlab
         delegate :count, :size, :real_size, to: :raw_diff_files
 
         def self.default_options
-          ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false, include_stats: true)
+          ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false, include_stats: true, use_extra_viewer_as_main: false)
         end
 
         def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil)
@@ -25,6 +25,7 @@ module Gitlab
           @diff_refs = diff_refs
           @fallback_diff_refs = fallback_diff_refs
           @repository = project.repository
+          @use_extra_viewer_as_main = diff_options.delete(:use_extra_viewer_as_main)
         end
 
         def diffs
@@ -120,11 +121,17 @@ module Gitlab
 
           stats = diff_stats_collection&.find_by_path(diff.new_path)
 
-          Gitlab::Diff::File.new(diff,
+          diff_file = Gitlab::Diff::File.new(diff,
                                  repository: project.repository,
                                  diff_refs: diff_refs,
                                  fallback_diff_refs: fallback_diff_refs,
                                  stats: stats)
+
+          if @use_extra_viewer_as_main && diff_file.has_renderable?
+            diff_file.rendered
+          else
+            diff_file
+          end
         end
 
         def sort_diffs(diffs)
diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb
index b459e3f6619..c6ab56e783a 100644
--- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb
+++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb
@@ -12,10 +12,11 @@ module Gitlab
           @merge_request_diff = merge_request_diff
 
           super(merge_request_diff,
-            project: merge_request_diff.project,
-            diff_options: diff_options,
-            diff_refs: merge_request_diff.diff_refs,
-            fallback_diff_refs: merge_request_diff.fallback_diff_refs)
+                project: merge_request_diff.project,
+                diff_options: diff_options,
+                diff_refs: merge_request_diff.diff_refs,
+                fallback_diff_refs: merge_request_diff.fallback_diff_refs
+          )
         end
 
         def diff_files(sorted: false)
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index 6cf414e29cc..c2b834c71b5 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -8,9 +8,9 @@ module Gitlab
       #
       SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze
 
-      attr_reader :line_code, :marker_ranges
-      attr_writer :text, :rich_text
-      attr_accessor :index, :type, :old_pos, :new_pos
+      attr_reader :marker_ranges
+      attr_writer :text, :rich_text, :discussable
+      attr_accessor :index, :type, :old_pos, :new_pos, :line_code
 
       def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil)
         @text = text
@@ -26,6 +26,7 @@ module Gitlab
         @line_code = line_code || calculate_line_code
 
         @marker_ranges = []
+        @discussable = true
       end
 
       def self.init_from_hash(hash)
@@ -96,7 +97,7 @@ module Gitlab
       end
 
       def discussable?
-        !meta?
+        @discussable && !meta?
       end
 
       def suggestible?
diff --git a/lib/gitlab/diff/rendered/notebook/diff_file.rb b/lib/gitlab/diff/rendered/notebook/diff_file.rb
new file mode 100644
index 00000000000..8e76d753dc6
--- /dev/null
+++ b/lib/gitlab/diff/rendered/notebook/diff_file.rb
@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+module Gitlab
+  module Diff
+    module Rendered
+      module Notebook
+        include Gitlab::Utils::StrongMemoize
+
+        class DiffFile < Gitlab::Diff::File
+          attr_reader :source_diff
+
+          delegate :repository, :diff_refs, :fallback_diff_refs, :unfolded, :unique_identifier,
+                   to: :source_diff
+
+          def initialize(diff_file)
+            @source_diff = diff_file
+          end
+
+          def old_blob
+            return unless notebook_diff
+
+            strong_memoize(:old_blob) { ::Blobs::Notebook.decorate(source_diff.old_blob, notebook_diff.from.as_text) }
+          end
+
+          def new_blob
+            return unless notebook_diff
+
+            strong_memoize(:new_blob) { ::Blobs::Notebook.decorate(source_diff.new_blob, notebook_diff.to.as_text) }
+          end
+
+          def diff
+            strong_memoize(:diff) { transformed_diff }
+          end
+
+          def has_renderable?
+            !notebook_diff.nil? && diff.diff.present?
+          end
+
+          def rendered
+            self
+          end
+
+          def highlighted_diff_lines
+            @highlighted_diff_lines ||= begin
+              removal_line_maps, addition_line_maps = compute_end_start_map
+              Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight.map do |line|
+                mutate_line(line, addition_line_maps, removal_line_maps)
+              end
+            end
+          end
+
+          private
+
+          def notebook_diff
+            strong_memoize(:notebook_diff) do
+              Gitlab::AppLogger.info({ message: 'IPYNB_DIFF_GENERATED' })
+
+              IpynbDiff.diff(source_diff.old_blob&.data, source_diff.new_blob&.data,
+                             raise_if_invalid_nb: true,
+                             diffy_opts: { include_diff_info: true })
+            rescue IpynbDiff::InvalidNotebookError => e
+              Gitlab::ErrorTracking.log_exception(e)
+              nil
+            end
+          end
+
+          def transformed_diff
+            return unless notebook_diff
+
+            diff = source_diff.diff.clone
+            diff.diff = strip_diff_frontmatter(notebook_diff.to_s(:text))
+            diff
+          end
+
+          def strip_diff_frontmatter(diff_content)
+            diff_content.scan(/.*\n/)[2..]&.join('') if diff_content.present?
+          end
+
+          def transformed_line_to_source(transformed_line, transformed_blocks)
+            transformed_blocks.empty? ? 0 : ( transformed_blocks[transformed_line - 1][:source_line] || -1 ) + 1
+          end
+
+          def mutate_line(line, addition_line_maps, removal_line_maps)
+            line.new_pos = transformed_line_to_source(line.new_pos, notebook_diff.to.blocks)
+            line.old_pos = transformed_line_to_source(line.old_pos, notebook_diff.from.blocks)
+
+            line.old_pos = addition_line_maps[line.new_pos] if line.old_pos == 0 && line.new_pos != 0
+            line.new_pos = removal_line_maps[line.old_pos] if line.new_pos == 0 && line.old_pos != 0
+
+            # Lines that do not appear on the original diff should not be commentable
+
+            unless addition_line_maps[line.new_pos] || removal_line_maps[line.old_pos]
+              line.discussable = false
+            end
+
+            line.line_code = line_code(line)
+            line
+          end
+
+          def compute_end_start_map
+            # line_codes are used for assigning notes to diffs, and these depend on the line on the new version and the
+            # line that would have been that one in the previous version. However, since we do a transformation on the
+            # file, that map gets lost. To overcome this, we look at the original source lines and build two maps:
+            # - For additions, we look at the latest line change for that line and pick the old line for that id
+            # - For removals, we look at the first line in the old version, and pick the first line on the new version
+            #
+            #
+            # The caveat here is that we can't have notes on lines that are not a translation of a line in the source
+            # diff
+            #
+            # (gitlab/diff/file.rb:75)
+
+            removals = {}
+            additions = {}
+
+            source_diff.highlighted_diff_lines.each do |line|
+              removals[line.old_pos] = line.new_pos
+              additions[line.new_pos] = line.old_pos
+            end
+
+            [removals, additions]
+          end
+        end
+      end
+    end
+  end
+end