Load and process at most 100 commits when pushing into default branchdm-large-push-performance

3 files changed, 28 insertions, 13 deletions

diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb
index 20d1fb29289..5db1cb21333 100644
--- a/app/services/git_push_service.rb
+++ b/app/services/git_push_service.rb
@@ -1,5 +1,5 @@
 class GitPushService < BaseService
-  attr_accessor :push_data, :push_commits
+  attr_accessor :push_data
   include Gitlab::CurrentSettings
   include Gitlab::Access
 
@@ -45,6 +45,7 @@ class GitPushService < BaseService
     elsif push_to_existing_branch?
       # Collect data for this git push
       @push_commits = @project.repository.commits_between(params[:oldrev], params[:newrev])
+
       process_commit_messages
 
       # Update the bare repositories info/attributes file using the contents of the default branches
@@ -64,15 +65,21 @@ class GitPushService < BaseService
 
   def update_caches
     if is_default_branch?
-      paths = Set.new
+      if push_to_new_branch?
+        # If this is the initial push into the default branch, the file type caches
+        # will already be reset as a result of `Project#change_head`.
+        types = []
+      else
+        paths = Set.new
 
-      @push_commits.each do |commit|
-        commit.raw_deltas.each do |diff|
-          paths << diff.new_path
+        @push_commits.last(PROCESS_COMMIT_LIMIT).each do |commit|
+          commit.raw_deltas.each do |diff|
+            paths << diff.new_path
+          end
         end
-      end
 
-      types = Gitlab::FileDetector.types_in_paths(paths.to_a)
+        types = Gitlab::FileDetector.types_in_paths(paths.to_a)
+      end
     else
       types = []
     end
@@ -84,7 +91,7 @@ class GitPushService < BaseService
   def process_commit_messages
     default = is_default_branch?
 
-    push_commits.last(PROCESS_COMMIT_LIMIT).each do |commit|
+    @push_commits.last(PROCESS_COMMIT_LIMIT).each do |commit|
       if commit.matches_cross_reference_regex?
         ProcessCommitWorker
           .perform_async(project.id, current_user.id, commit.to_hash, default)
@@ -103,7 +110,7 @@ class GitPushService < BaseService
 
     EventCreateService.new.push(@project, current_user, build_push_data)
     Ci::CreatePipelineService.new(@project, current_user, build_push_data).execute(:push)
-    
+
     SystemHookPushWorker.perform_async(build_push_data.dup, :push_hooks)
     @project.execute_hooks(build_push_data.dup, :push_hooks)
     @project.execute_services(build_push_data.dup, :push_hooks)
@@ -123,7 +130,10 @@ class GitPushService < BaseService
   end
 
   def process_default_branch
-    @push_commits = project.repository.commits(params[:newrev])
+    @push_commits_count = project.repository.commit_count_for_ref(params[:ref])
+
+    offset = [@push_commits_count - PROCESS_COMMIT_LIMIT, 0].max
+    @push_commits = project.repository.commits(params[:newrev], offset: offset, limit: PROCESS_COMMIT_LIMIT)
 
     # Ensure HEAD points to the default branch in case it is not master
     project.change_head(branch_name)
@@ -152,7 +162,8 @@ class GitPushService < BaseService
       params[:oldrev],
       params[:newrev],
       params[:ref],
-      push_commits)
+      @push_commits,
+      commits_count: @push_commits_count)
   end
 
   def push_to_existing_branch?
diff --git a/changelogs/unreleased/dm-large-push-performance.yml b/changelogs/unreleased/dm-large-push-performance.yml
new file mode 100644
index 00000000000..f5fe1bd3b28
--- /dev/null
+++ b/changelogs/unreleased/dm-large-push-performance.yml
@@ -0,0 +1,4 @@
+---
+title: Improve performance of large (initial) push into default branch
+merge_request:
+author:
diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb
index 8c8729b6557..5c5f507d44d 100644
--- a/lib/gitlab/data_builder/push.rb
+++ b/lib/gitlab/data_builder/push.rb
@@ -24,11 +24,11 @@ module Gitlab
       #   total_commits_count: Fixnum
       # }
       #
-      def build(project, user, oldrev, newrev, ref, commits = [], message = nil)
+      def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil)
         commits = Array(commits)
 
         # Total commits count
-        commits_count = commits.size
+        commits_count ||= commits.size
 
         # Get latest 20 commits ASC
         commits_limited = commits.last(20)