From 8ceea56c143e6144ac62b20b22fcd734104400a8 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 30 Mar 2018 16:32:21 +0200 Subject: Export LFS Objects when exporting a project The LFS files will be included in the `lfs-objects` directory in the archive. --- .../projects/import_export/export_service.rb | 6 ++- app/views/projects/_export.html.haml | 2 +- doc/user/project/settings/import_export.md | 2 +- lib/gitlab/import_export/lfs_saver.rb | 48 ++++++++++++++++++++++ spec/lib/gitlab/import_export/lfs_saver_spec.rb | 39 ++++++++++++++++++ .../projects/import_export/export_service_spec.rb | 43 +++++++++++++++++++ 6 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 lib/gitlab/import_export/lfs_saver.rb create mode 100644 spec/lib/gitlab/import_export/lfs_saver_spec.rb diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb index 402cddd3ec1..7bf0b90b491 100644 --- a/app/services/projects/import_export/export_service.rb +++ b/app/services/projects/import_export/export_service.rb @@ -28,7 +28,7 @@ module Projects end def save_services - [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save) + [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver].all?(&:save) end def version_saver @@ -55,6 +55,10 @@ module Projects Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared) end + def lfs_saver + Gitlab::ImportExport::LfsSaver.new(project: project, shared: @shared) + end + def cleanup_and_notify_error Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{@shared.errors.join(', ')}") diff --git a/app/views/projects/_export.html.haml b/app/views/projects/_export.html.haml index 825bfd0707f..1e7d9444986 100644 --- a/app/views/projects/_export.html.haml +++ b/app/views/projects/_export.html.haml @@ -21,11 +21,11 @@ %li Project uploads %li Project configuration including web hooks and services %li Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities + %li LFS objects %p The following items will NOT be exported: %ul %li Job traces and artifacts - %li LFS objects %li Container registry images %li CI variables %li Any encrypted tokens diff --git a/doc/user/project/settings/import_export.md b/doc/user/project/settings/import_export.md index dedf102fc37..eb0ac221e30 100644 --- a/doc/user/project/settings/import_export.md +++ b/doc/user/project/settings/import_export.md @@ -57,11 +57,11 @@ The following items will be exported: - Project configuration including web hooks and services - Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities +- LFS objects The following items will NOT be exported: - Build traces and artifacts -- LFS objects - Container registry images - CI variables - Any encrypted tokens diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb new file mode 100644 index 00000000000..bb7a070fe15 --- /dev/null +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -0,0 +1,48 @@ +module Gitlab + module ImportExport + class LfsSaver + include Gitlab::ImportExport::CommandLineUtil + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return true if @project.lfs_objects.empty? + + @project.lfs_objects.each do |lfs_object| + save_lfs_object(lfs_object) + end + + true + rescue => e + @shared.error(e) + + false + end + + private + + def save_lfs_object(lfs_object) + if lfs_object.local_store? + copy_file_for_lfs_object(lfs_object) + else + raise NotImplementedError.new "Exporting files from object storage is not yet supported" + end + end + + def copy_file_for_lfs_object(lfs_object) + copy_files(lfs_object.file.path, destination_path_for_object(lfs_object)) + end + + def destination_path_for_object(lfs_object) + File.join(lfs_export_path, lfs_object.oid) + end + + def lfs_export_path + File.join(@shared.export_path, 'lfs-objects') + end + end + end +end diff --git a/spec/lib/gitlab/import_export/lfs_saver_spec.rb b/spec/lib/gitlab/import_export/lfs_saver_spec.rb new file mode 100644 index 00000000000..e2237cd22cf --- /dev/null +++ b/spec/lib/gitlab/import_export/lfs_saver_spec.rb @@ -0,0 +1,39 @@ +require 'spec_helper' + +describe Gitlab::ImportExport::LfsSaver do + let(:shared) { project.import_export_shared } + let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } + let(:project) { create(:project) } + + subject(:saver) { described_class.new(project: project, shared: shared) } + + before do + allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) + FileUtils.mkdir_p(shared.export_path) + end + + after do + FileUtils.rm_rf(shared.export_path) + end + + describe '#save' do + context 'when the project has LFS objects' do + let(:lfs_object) { create(:lfs_object, :with_file) } + before do + project.lfs_objects << lfs_object\ + end + + it 'does not cause errors' do + saver.save + + expect(shared.errors).to be_empty + end + + it 'copies the file in the correct location when there is an lfs object' do + saver.save + + expect(File).to exist("#{shared.export_path}/lfs-objects/#{lfs_object.oid}") + end + end + end +end diff --git a/spec/services/projects/import_export/export_service_spec.rb b/spec/services/projects/import_export/export_service_spec.rb index 51491c7d529..f9e5530bc9d 100644 --- a/spec/services/projects/import_export/export_service_spec.rb +++ b/spec/services/projects/import_export/export_service_spec.rb @@ -8,6 +8,49 @@ describe Projects::ImportExport::ExportService do let(:service) { described_class.new(project, user) } let!(:after_export_strategy) { Gitlab::ImportExport::AfterExportStrategies::DownloadNotificationStrategy.new } + it 'saves the version' do + expect(Gitlab::ImportExport::VersionSaver).to receive(:new).and_call_original + + service.execute + end + + it 'saves the avatar' do + expect(Gitlab::ImportExport::AvatarSaver).to receive(:new).and_call_original + + service.execute + end + + it 'saves the models' do + expect(Gitlab::ImportExport::ProjectTreeSaver).to receive(:new).and_call_original + + service.execute + end + + it 'saves the uploads' do + expect(Gitlab::ImportExport::UploadsSaver).to receive(:new).and_call_original + + service.execute + end + + it 'saves the repo' do + # once for the normal repo, once for the wiki + expect(Gitlab::ImportExport::RepoSaver).to receive(:new).twice.and_call_original + + service.execute + end + + it 'saves the lfs objects' do + expect(Gitlab::ImportExport::LfsSaver).to receive(:new).and_call_original + + service.execute + end + + it 'saves the wiki repo' do + expect(Gitlab::ImportExport::WikiRepoSaver).to receive(:new).and_call_original + + service.execute + end + context 'when all saver services succeed' do before do allow(service).to receive(:save_services).and_return(true) -- cgit v1.2.1