Merge branch 'sh-lfs-object-batches' into 'master'

Makes LFS object link process OIDs in batches

Closes #66274

See merge request gitlab-org/gitlab-ce!32268

2 files changed, 25 insertions, 5 deletions

diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb
index 79a376ff0fd..40695a97d97 100644
--- a/app/models/lfs_object.rb
+++ b/app/models/lfs_object.rb
@@ -2,6 +2,7 @@
 
 class LfsObject < ApplicationRecord
   include AfterCommitQueue
+  include EachBatch
   include ObjectStorage::BackgroundMove
 
   has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
diff --git a/app/services/projects/lfs_pointers/lfs_link_service.rb b/app/services/projects/lfs_pointers/lfs_link_service.rb
index e3c956250f0..38de2af9c1e 100644
--- a/app/services/projects/lfs_pointers/lfs_link_service.rb
+++ b/app/services/projects/lfs_pointers/lfs_link_service.rb
@@ -4,6 +4,8 @@
 module Projects
   module LfsPointers
     class LfsLinkService < BaseService
+      BATCH_SIZE = 1000
+
       # Accept an array of oids to link
       #
       # Returns an array with the oid of the existent lfs objects
@@ -18,16 +20,33 @@ module Projects
 
       # rubocop: disable CodeReuse/ActiveRecord
       def link_existing_lfs_objects(oids)
-        existent_lfs_objects = LfsObject.where(oid: oids)
+        all_existing_objects = []
+        iterations = 0
+
+        LfsObject.where(oid: oids).each_batch(of: BATCH_SIZE) do |existent_lfs_objects|
+          next unless existent_lfs_objects.any?
+
+          iterations += 1
+          not_linked_lfs_objects = existent_lfs_objects.where.not(id: project.all_lfs_objects)
+          project.all_lfs_objects << not_linked_lfs_objects
 
-        return [] unless existent_lfs_objects.any?
+          all_existing_objects += existent_lfs_objects.pluck(:oid)
+        end
 
-        not_linked_lfs_objects = existent_lfs_objects.where.not(id: project.all_lfs_objects)
-        project.all_lfs_objects << not_linked_lfs_objects
+        log_lfs_link_results(all_existing_objects.count, iterations)
 
-        existent_lfs_objects.pluck(:oid)
+        all_existing_objects
       end
       # rubocop: enable CodeReuse/ActiveRecord
+
+      def log_lfs_link_results(lfs_objects_linked_count, iterations)
+        Gitlab::Import::Logger.info(
+          class: self.class.name,
+          project_id: project.id,
+          project_path: project.full_path,
+          lfs_objects_linked_count: lfs_objects_linked_count,
+          iterations: iterations)
+      end
     end
   end
 end