Merge branch 'backport-of-rd-3429-enabling-maximum-file-size-limit-in-repository-causes-pushes-to-fail' into 'master'

Backport of EE !4989

See merge request gitlab-org/gitlab-ce!18238

6 files changed, 139 insertions, 7 deletions

diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index d4e893b881c..c9abea90d21 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -1,5 +1,9 @@
 module Gitlab
   module Git
+    # The ID of empty tree.
+    # See http://stackoverflow.com/a/40884093/1856239 and
+    # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012
+    EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze
     BLANK_SHA = ('0' * 40).freeze
     TAG_REF_PREFIX = "refs/tags/".freeze
     BRANCH_REF_PREFIX = "refs/heads/".freeze
diff --git a/lib/gitlab/git/raw_diff_change.rb b/lib/gitlab/git/raw_diff_change.rb
new file mode 100644
index 00000000000..eb3d8819239
--- /dev/null
+++ b/lib/gitlab/git/raw_diff_change.rb
@@ -0,0 +1,60 @@
+module Gitlab
+  module Git
+    # This class behaves like a struct with fields :blob_id, :blob_size, :operation, :old_path, :new_path
+    # All those fields are (binary) strings or integers
+    class RawDiffChange
+      attr_reader :blob_id, :blob_size, :old_path, :new_path, :operation
+
+      def initialize(raw_change)
+        parse(raw_change)
+      end
+
+      private
+
+      # Input data has the following format:
+      #
+      # When a file has been modified:
+      # 7e3e39ebb9b2bf433b4ad17313770fbe4051649c 669 M\tfiles/ruby/popen.rb
+      #
+      # When a file has been renamed:
+      # 85bc2f9753afd5f4fc5d7c75f74f8d526f26b4f3 107 R060\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee
+      def parse(raw_change)
+        @blob_id, @blob_size, @raw_operation, raw_paths = raw_change.split(' ', 4)
+        @operation = extract_operation
+        @old_path, @new_path = extract_paths(raw_paths)
+      end
+
+      def extract_paths(file_path)
+        case operation
+        when :renamed
+          file_path.split(/\t/)
+        when :deleted
+          [file_path, nil]
+        when :added
+          [nil, file_path]
+        else
+          [file_path, file_path]
+        end
+      end
+
+      def extract_operation
+        case @raw_operation&.first(1)
+        when 'A'
+          :added
+        when 'C'
+          :copied
+        when 'D'
+          :deleted
+        when 'M'
+          :modified
+        when 'R'
+          :renamed
+        when 'T'
+          :type_changed
+        else
+          :unknown
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 1a0a793564e..36992cbcca0 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -558,6 +558,24 @@ module Gitlab
         count_commits(from: from, to: to, **options)
       end
 
+      # old_rev and new_rev are commit ID's
+      # the result of this method is an array of Gitlab::Git::RawDiffChange
+      def raw_changes_between(old_rev, new_rev)
+        result = []
+
+        circuit_breaker.perform do
+          Open3.pipeline_r(git_diff_cmd(old_rev, new_rev), format_git_cat_file_script, git_cat_file_cmd) do |last_stdout, wait_threads|
+            last_stdout.each_line { |line| result << ::Gitlab::Git::RawDiffChange.new(line.chomp!) }
+
+            if wait_threads.any? { |waiter| !waiter.value&.success? }
+              raise ::Gitlab::Git::Repository::GitError, "Unabled to obtain changes between #{old_rev} and #{new_rev}"
+            end
+          end
+        end
+
+        result
+      end
+
       # Returns the SHA of the most recent common ancestor of +from+ and +to+
       def merge_base(from, to)
         gitaly_migrate(:merge_base) do |is_enabled|
@@ -2483,6 +2501,35 @@ module Gitlab
 
         result.to_s(16)
       end
+
+      def build_git_cmd(*args)
+        object_directories = alternate_object_directories.join(File::PATH_SEPARATOR)
+
+        env = { 'PWD' => self.path }
+        env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories if object_directories.present?
+
+        [
+          env,
+          ::Gitlab.config.git.bin_path,
+          *args,
+          { chdir: self.path }
+        ]
+      end
+
+      def git_diff_cmd(old_rev, new_rev)
+        old_rev = old_rev == ::Gitlab::Git::BLANK_SHA ? ::Gitlab::Git::EMPTY_TREE_ID : old_rev
+
+        build_git_cmd('diff', old_rev, new_rev, '--raw')
+      end
+
+      def git_cat_file_cmd
+        format = '%(objectname) %(objectsize) %(rest)'
+        build_git_cmd('cat-file', "--batch-check=#{format}")
+      end
+
+      def format_git_cat_file_script
+        File.expand_path('../support/format-git-cat-file-input', __FILE__)
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/support/format-git-cat-file-input b/lib/gitlab/git/support/format-git-cat-file-input
new file mode 100755
index 00000000000..2e93c646d0f
--- /dev/null
+++ b/lib/gitlab/git/support/format-git-cat-file-input
@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+# This script formats the output of the `git diff <old_rev> <new_rev> --raw`
+# command so it can be processed by `git cat-file`
+
+# We need to convert this:
+# ":100644 100644 5f53439... 85bc2f9... R060\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee"
+# To:
+# "85bc2f9 R\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee"
+
+ARGF.each do |line|
+  _, _, old_blob_id, new_blob_id, rest = line.split(/\s/, 5)
+
+  old_blob_id.gsub!(/[^\h]/, '')
+  new_blob_id.gsub!(/[^\h]/, '')
+
+  # We can't pass '0000000...' to `git cat-file` given it will not return info about the deleted file
+  blob_id = new_blob_id =~ /\A0+\z/ ? old_blob_id : new_blob_id
+
+  $stdout.puts "#{blob_id} #{rest}"
+end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 456a8a1a2d6..a36e6c822f9 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -3,10 +3,6 @@ module Gitlab
     class CommitService
       include Gitlab::EncodingHelper
 
-      # The ID of empty tree.
-      # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012
-      EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze
-
       def initialize(repository)
         @gitaly_repo = repository.gitaly_repository
         @repository = repository
@@ -37,7 +33,7 @@ module Gitlab
       def diff(from, to, options = {})
         from_id = case from
                   when NilClass
-                    EMPTY_TREE_ID
+                    Gitlab::Git::EMPTY_TREE_ID
                   else
                     if from.respond_to?(:oid)
                       # This is meant to match a Rugged::Commit. This should be impossible in
@@ -50,7 +46,7 @@ module Gitlab
 
         to_id = case to
                 when NilClass
-                  EMPTY_TREE_ID
+                  Gitlab::Git::EMPTY_TREE_ID
                 else
                   if to.respond_to?(:oid)
                     # This is meant to match a Rugged::Commit. This should be impossible in
@@ -352,7 +348,7 @@ module Gitlab
       end
 
       def diff_from_parent_request_params(commit, options = {})
-        parent_id = commit.parent_ids.first || EMPTY_TREE_ID
+        parent_id = commit.parent_ids.first || Gitlab::Git::EMPTY_TREE_ID
 
         diff_between_commits_request_params(parent_id, commit.id, options)
       end
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index b0a492eaa58..aeda66763e8 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -73,6 +73,10 @@ module Gitlab
       nil
     end
 
+    def bytes_to_megabytes(bytes)
+      bytes.to_f / Numeric::MEGABYTE
+    end
+
     # Used in EE
     # Accepts either an Array or a String and returns an array
     def ensure_array_from_string(string_or_array)