From 79cb4d99c0e47bfd988788ff38871e668367dfbf Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 30 Mar 2018 19:45:58 +0200 Subject: Import projects with LFS objects If the LFS object already exist, we'll link it tot he existing one, if not we'll create it. --- lib/gitlab/import_export/importer.rb | 11 ++- lib/gitlab/import_export/lfs_restorer.rb | 43 ++++++++++++ lib/gitlab/import_export/lfs_saver.rb | 2 +- spec/fixtures/exported-project.gz | Bin 0 -> 2306 bytes spec/lib/gitlab/import_export/importer_spec.rb | 64 ++++++++++++++++++ spec/lib/gitlab/import_export/lfs_restorer_spec.rb | 75 +++++++++++++++++++++ spec/lib/gitlab/import_export/lfs_saver_spec.rb | 3 +- 7 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 lib/gitlab/import_export/lfs_restorer.rb create mode 100644 spec/fixtures/exported-project.gz create mode 100644 spec/lib/gitlab/import_export/importer_spec.rb create mode 100644 spec/lib/gitlab/import_export/lfs_restorer_spec.rb diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index c38df9102eb..c490bf059d2 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -13,7 +13,7 @@ module Gitlab end def execute - if import_file && check_version! && [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer].all?(&:restore) + if import_file && check_version! && restorers.all?(&:restore) project_tree.restored_project else raise Projects::ImportService::Error.new(@shared.errors.join(', ')) @@ -24,6 +24,11 @@ module Gitlab private + def restorers + [repo_restorer, wiki_restorer, project_tree, avatar_restorer, + uploads_restorer, lfs_restorer] + end + def import_file Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, shared: @shared) @@ -60,6 +65,10 @@ module Gitlab Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) end + def lfs_restorer + Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared) + end + def path_with_namespace File.join(@project.namespace.full_path, @project.path) end diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb new file mode 100644 index 00000000000..4f144d5c8e6 --- /dev/null +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -0,0 +1,43 @@ +module Gitlab + module ImportExport + class LfsRestorer + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def restore + return true if lfs_file_paths.empty? + + lfs_file_paths.each do |file_path| + link_or_create_lfs_object!(file_path) + end + + true + rescue => e + @shared.error(e) + false + end + + private + + def link_or_create_lfs_object!(path) + size = File.size(path) + oid = LfsObject.calculate_oid(path) + + lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size) + lfs_object.file = File.open(path) unless lfs_object.file&.exists? + + @project.lfs_storage_project.lfs_objects << lfs_object + end + + def lfs_file_paths + @lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*") + end + + def lfs_storage_path + File.join(@shared.export_path, 'lfs-objects') + end + end + end +end diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index bb7a070fe15..d796440902b 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -11,7 +11,7 @@ module Gitlab def save return true if @project.lfs_objects.empty? - @project.lfs_objects.each do |lfs_object| + @project.lfs_storage_project.lfs_objects.each do |lfs_object| save_lfs_object(lfs_object) end diff --git a/spec/fixtures/exported-project.gz b/spec/fixtures/exported-project.gz new file mode 100644 index 00000000000..352384f16c8 Binary files /dev/null and b/spec/fixtures/exported-project.gz differ diff --git a/spec/lib/gitlab/import_export/importer_spec.rb b/spec/lib/gitlab/import_export/importer_spec.rb new file mode 100644 index 00000000000..d75416f2a62 --- /dev/null +++ b/spec/lib/gitlab/import_export/importer_spec.rb @@ -0,0 +1,64 @@ +require 'spec_helper' + +describe Gitlab::ImportExport::Importer do + let(:test_path) { "#{Dir.tmpdir}/importer_spec" } + let(:shared) { project.import_export_shared } + let(:project) { create(:project, import_source: File.join(test_path, 'exported-project.gz')) } + + subject(:importer) { described_class.new(project) } + + before do + allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path) + FileUtils.mkdir_p(shared.export_path) + FileUtils.cp(Rails.root.join('spec', 'fixtures', 'exported-project.gz'), test_path) + end + + after do + FileUtils.rm_rf(test_path) + end + + describe '#execute' do + it 'succeeds' do + importer.execute + + expect(shared.errors).to be_empty + end + + it 'extracts the archive' do + expect(Gitlab::ImportExport::FileImporter).to receive(:import).and_call_original + + importer.execute + end + + it 'checks the version' do + expect(Gitlab::ImportExport::VersionChecker).to receive(:check!).and_call_original + + importer.execute + end + + context 'all restores are executed' do + [ + Gitlab::ImportExport::AvatarRestorer, + Gitlab::ImportExport::RepoRestorer, + Gitlab::ImportExport::WikiRestorer, + Gitlab::ImportExport::UploadsRestorer, + Gitlab::ImportExport::LfsRestorer + ].each do |restorer| + it "calls the #{restorer}" do + fake_restorer = double(restorer.to_s) + + expect(fake_restorer).to receive(:restore).and_return(true).at_least(1) + expect(restorer).to receive(:new).and_return(fake_restorer).at_least(1) + + importer.execute + end + end + + it 'restores the ProjectTree' do + expect(Gitlab::ImportExport::ProjectTreeRestorer).to receive(:new).and_call_original + + importer.execute + end + end + end +end diff --git a/spec/lib/gitlab/import_export/lfs_restorer_spec.rb b/spec/lib/gitlab/import_export/lfs_restorer_spec.rb new file mode 100644 index 00000000000..70eeb9ee66b --- /dev/null +++ b/spec/lib/gitlab/import_export/lfs_restorer_spec.rb @@ -0,0 +1,75 @@ +require 'spec_helper' + +describe Gitlab::ImportExport::LfsRestorer do + include UploadHelpers + + let(:export_path) { "#{Dir.tmpdir}/lfs_object_restorer_spec" } + let(:project) { create(:project) } + let(:shared) { project.import_export_shared } + subject(:restorer) { described_class.new(project: project, shared: shared) } + + before do + allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) + FileUtils.mkdir_p(shared.export_path) + end + + after do + FileUtils.rm_rf(shared.export_path) + end + + describe '#restore' do + context 'when the archive contains lfs files' do + let(:dummy_lfs_file_path) { File.join(shared.export_path, 'lfs-objects', 'dummy') } + + def create_lfs_object_with_content(content) + dummy_lfs_file = Tempfile.new('existing') + File.write(dummy_lfs_file.path, content) + size = dummy_lfs_file.size + oid = LfsObject.calculate_oid(dummy_lfs_file.path) + LfsObject.create!(oid: oid, size: size, file: dummy_lfs_file) + end + + before do + FileUtils.mkdir_p(File.dirname(dummy_lfs_file_path)) + File.write(dummy_lfs_file_path, 'not very large') + allow(restorer).to receive(:lfs_file_paths).and_return([dummy_lfs_file_path]) + end + + it 'creates an lfs object for the project' do + expect { restorer.restore }.to change { project.reload.lfs_objects.size }.by(1) + end + + it 'assigns the file correctly' do + restorer.restore + + expect(project.lfs_objects.first.file.read).to eq('not very large') + end + + it 'links an existing LFS object if it existed' do + lfs_object = create_lfs_object_with_content('not very large') + + restorer.restore + + expect(project.lfs_objects).to include(lfs_object) + end + + it 'succeeds' do + expect(restorer.restore).to be_truthy + expect(shared.errors).to be_empty + end + + it 'stores the upload' do + expect_any_instance_of(LfsObjectUploader).to receive(:store!) + + restorer.restore + end + end + + context 'without any LFS-objects' do + it 'succeeds' do + expect(restorer.restore).to be_truthy + expect(shared.errors).to be_empty + end + end + end +end diff --git a/spec/lib/gitlab/import_export/lfs_saver_spec.rb b/spec/lib/gitlab/import_export/lfs_saver_spec.rb index e2237cd22cf..e62afac1c48 100644 --- a/spec/lib/gitlab/import_export/lfs_saver_spec.rb +++ b/spec/lib/gitlab/import_export/lfs_saver_spec.rb @@ -19,8 +19,9 @@ describe Gitlab::ImportExport::LfsSaver do describe '#save' do context 'when the project has LFS objects' do let(:lfs_object) { create(:lfs_object, :with_file) } + before do - project.lfs_objects << lfs_object\ + project.lfs_objects << lfs_object end it 'does not cause errors' do -- cgit v1.2.1