diff options
author | Luke Duncalfe <lduncalfe@gitlab.com> | 2019-07-24 11:23:51 +0000 |
---|---|---|
committer | James Lopez <james@gitlab.com> | 2019-07-24 11:23:51 +0000 |
commit | d18ee3faad34097c14d2f1ee9c4a1bf7b00b202a (patch) | |
tree | 1b97dbef9e1d53eba6208831f44ec8119c05bc07 /lib/gitlab/import_export/lfs_saver.rb | |
parent | 0a2bbc9267e5aca220cda17f13e6f62bae7f1f1b (diff) | |
download | gitlab-ce-d18ee3faad34097c14d2f1ee9c4a1bf7b00b202a.tar.gz |
LFS export records repository_type data
A project can have the same `LfsObject` linked with up to three
`LfsObjectsProject` records. Each of these records would be for a
different repository, recorded in the `repository_type` property. The
different repositories at time of writing are "project", "wiki", and
"design". See https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/13894
This change exports the list of `repository_type`s as a JSON mapping of
oid => repository_types, which are imported to recreate the correct
`LfsObjectsProject` records.
https://gitlab.com/gitlab-org/gitlab-ee/issues/11090
Diffstat (limited to 'lib/gitlab/import_export/lfs_saver.rb')
-rw-r--r-- | lib/gitlab/import_export/lfs_saver.rb | 47 |
1 files changed, 43 insertions, 4 deletions
diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 954f6f00078..18c590e1ca9 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -5,25 +5,40 @@ module Gitlab class LfsSaver include Gitlab::ImportExport::CommandLineUtil + attr_accessor :lfs_json, :project, :shared + + BATCH_SIZE = 100 + def initialize(project:, shared:) @project = project @shared = shared + @lfs_json = {} end def save - @project.all_lfs_objects.each do |lfs_object| - save_lfs_object(lfs_object) + project.all_lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| + batch.each do |lfs_object| + save_lfs_object(lfs_object) + end + + append_lfs_json_for_batch(batch) if write_lfs_json_enabled? end + write_lfs_json if write_lfs_json_enabled? + true rescue => e - @shared.error(e) + shared.error(e) false end private + def write_lfs_json_enabled? + ::Feature.enabled?(:export_lfs_objects_projects, default_enabled: true) + end + def save_lfs_object(lfs_object) if lfs_object.local_store? copy_file_for_lfs_object(lfs_object) @@ -45,12 +60,36 @@ module Gitlab copy_files(lfs_object.file.path, destination_path_for_object(lfs_object)) end + def append_lfs_json_for_batch(lfs_objects_batch) + lfs_objects_projects = LfsObjectsProject + .select('lfs_objects.oid, array_agg(distinct lfs_objects_projects.repository_type) as repository_types') + .joins(:lfs_object) + .where(project: project, lfs_object: lfs_objects_batch) + .group('lfs_objects.oid') + + lfs_objects_projects.each do |group| + oid = group.oid + + lfs_json[oid] ||= [] + lfs_json[oid] += group.repository_types + end + end + + def write_lfs_json + mkdir_p(shared.export_path) + File.write(lfs_json_path, lfs_json.to_json) + end + def destination_path_for_object(lfs_object) File.join(lfs_export_path, lfs_object.oid) end def lfs_export_path - File.join(@shared.export_path, 'lfs-objects') + File.join(shared.export_path, ImportExport.lfs_objects_storage) + end + + def lfs_json_path + File.join(shared.export_path, ImportExport.lfs_objects_filename) end end end |