summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRémy Coutable <remy@rymai.me>2019-08-28 11:14:23 +0000
committerRémy Coutable <remy@rymai.me>2019-08-28 11:14:23 +0000
commite8bcabb4a735ecb84b5bc65269fffc21a72b6da6 (patch)
tree3048d28f54da0a90b096401c4e81d578972960a4
parenta3b462e92a94f8647e00d3a8abe490b77f3b45ed (diff)
parent2022e6799bcbf119ea80145e4993ffdb7bb108e3 (diff)
downloadgitlab-ce-e8bcabb4a735ecb84b5bc65269fffc21a72b6da6.tar.gz
Merge branch 'sh-lfs-object-batches' into 'master'
Makes LFS object link process OIDs in batches Closes #66274 See merge request gitlab-org/gitlab-ce!32268
-rw-r--r--app/models/lfs_object.rb1
-rw-r--r--app/services/projects/lfs_pointers/lfs_link_service.rb29
-rw-r--r--changelogs/unreleased/sh-lfs-object-batches.yml5
-rw-r--r--spec/services/projects/lfs_pointers/lfs_link_service_spec.rb18
4 files changed, 48 insertions, 5 deletions
diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb
index 79a376ff0fd..40695a97d97 100644
--- a/app/models/lfs_object.rb
+++ b/app/models/lfs_object.rb
@@ -2,6 +2,7 @@
class LfsObject < ApplicationRecord
include AfterCommitQueue
+ include EachBatch
include ObjectStorage::BackgroundMove
has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
diff --git a/app/services/projects/lfs_pointers/lfs_link_service.rb b/app/services/projects/lfs_pointers/lfs_link_service.rb
index e3c956250f0..38de2af9c1e 100644
--- a/app/services/projects/lfs_pointers/lfs_link_service.rb
+++ b/app/services/projects/lfs_pointers/lfs_link_service.rb
@@ -4,6 +4,8 @@
module Projects
module LfsPointers
class LfsLinkService < BaseService
+ BATCH_SIZE = 1000
+
# Accept an array of oids to link
#
# Returns an array with the oid of the existent lfs objects
@@ -18,16 +20,33 @@ module Projects
# rubocop: disable CodeReuse/ActiveRecord
def link_existing_lfs_objects(oids)
- existent_lfs_objects = LfsObject.where(oid: oids)
+ all_existing_objects = []
+ iterations = 0
+
+ LfsObject.where(oid: oids).each_batch(of: BATCH_SIZE) do |existent_lfs_objects|
+ next unless existent_lfs_objects.any?
+
+ iterations += 1
+ not_linked_lfs_objects = existent_lfs_objects.where.not(id: project.all_lfs_objects)
+ project.all_lfs_objects << not_linked_lfs_objects
- return [] unless existent_lfs_objects.any?
+ all_existing_objects += existent_lfs_objects.pluck(:oid)
+ end
- not_linked_lfs_objects = existent_lfs_objects.where.not(id: project.all_lfs_objects)
- project.all_lfs_objects << not_linked_lfs_objects
+ log_lfs_link_results(all_existing_objects.count, iterations)
- existent_lfs_objects.pluck(:oid)
+ all_existing_objects
end
# rubocop: enable CodeReuse/ActiveRecord
+
+ def log_lfs_link_results(lfs_objects_linked_count, iterations)
+ Gitlab::Import::Logger.info(
+ class: self.class.name,
+ project_id: project.id,
+ project_path: project.full_path,
+ lfs_objects_linked_count: lfs_objects_linked_count,
+ iterations: iterations)
+ end
end
end
end
diff --git a/changelogs/unreleased/sh-lfs-object-batches.yml b/changelogs/unreleased/sh-lfs-object-batches.yml
new file mode 100644
index 00000000000..09043e286be
--- /dev/null
+++ b/changelogs/unreleased/sh-lfs-object-batches.yml
@@ -0,0 +1,5 @@
+---
+title: Makes LFS object linker process OIDs in batches
+merge_request: 32268
+author:
+type: performance
diff --git a/spec/services/projects/lfs_pointers/lfs_link_service_spec.rb b/spec/services/projects/lfs_pointers/lfs_link_service_spec.rb
index 849601c4a63..66233787d3a 100644
--- a/spec/services/projects/lfs_pointers/lfs_link_service_spec.rb
+++ b/spec/services/projects/lfs_pointers/lfs_link_service_spec.rb
@@ -30,5 +30,23 @@ describe Projects::LfsPointers::LfsLinkService do
expect(subject.execute(new_oid_list.keys)).to eq linked
end
+
+ it 'links in batches' do
+ stub_const("#{described_class}::BATCH_SIZE", 3)
+
+ expect(Gitlab::Import::Logger)
+ .to receive(:info)
+ .with(class: described_class.name,
+ project_id: project.id,
+ project_path: project.full_path,
+ lfs_objects_linked_count: 7,
+ iterations: 3)
+
+ lfs_objects = create_list(:lfs_object, 7)
+ linked = subject.execute(lfs_objects.pluck(:oid))
+
+ expect(project.all_lfs_objects.count).to eq 9
+ expect(linked.size).to eq 7
+ end
end
end