diff options
Diffstat (limited to 'lib/gitlab/import_export')
| -rw-r--r-- | lib/gitlab/import_export/avatar_saver.rb | 11 | ||||
| -rw-r--r-- | lib/gitlab/import_export/members_mapper.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/import_export/uploads_manager.rb | 101 | ||||
| -rw-r--r-- | lib/gitlab/import_export/uploads_restorer.rb | 21 | ||||
| -rw-r--r-- | lib/gitlab/import_export/uploads_saver.rb | 15 |
5 files changed, 131 insertions, 19 deletions
diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 998c21e2586..31ef0490cb3 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -11,7 +11,12 @@ module Gitlab def save return true unless @project.avatar.exists? - copy_files(avatar_path, avatar_export_path) + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared, + relative_export_path: 'avatar', + from: avatar_path + ).save rescue => e @shared.error(e) false @@ -19,10 +24,6 @@ module Gitlab private - def avatar_export_path - File.join(@shared.export_path, 'avatar', @project.avatar_identifier) - end - def avatar_path @project.avatar.path end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 8b8e48aac76..ac827cbe1ca 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -47,7 +47,7 @@ module Gitlab def ensure_default_member! @project.project_members.destroy_all - ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) + ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true) end def add_team_member(member, existing_user = nil) diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb new file mode 100644 index 00000000000..1110149712d --- /dev/null +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -0,0 +1,101 @@ +module Gitlab + module ImportExport + class UploadsManager + include Gitlab::ImportExport::CommandLineUtil + + UPLOADS_BATCH_SIZE = 100 + + def initialize(project:, shared:, relative_export_path: 'uploads', from: nil) + @project = project + @shared = shared + @relative_export_path = relative_export_path + @from = from || default_uploads_path + end + + def save + copy_files(@from, uploads_export_path) if File.directory?(@from) + + if File.file?(@from) && @relative_export_path == 'avatar' + copy_files(@from, File.join(uploads_export_path, @project.avatar.filename)) + end + + copy_from_object_storage + + true + rescue => e + @shared.error(e) + false + end + + def restore + Dir["#{uploads_export_path}/**/*"].each do |upload| + next if File.directory?(upload) + + add_upload(upload) + end + + true + rescue => e + @shared.error(e) + false + end + + private + + def add_upload(upload) + uploader_context = FileUploader.extract_dynamic_path(upload).named_captures.symbolize_keys + + UploadService.new(@project, File.open(upload, 'r'), FileUploader, uploader_context).execute + end + + def copy_from_object_storage + return unless Gitlab::ImportExport.object_storage? + + each_uploader do |uploader| + next unless uploader.file + next if uploader.upload.local? # Already copied, using the old method + + download_and_copy(uploader) + end + end + + def default_uploads_path + FileUploader.absolute_base_dir(@project) + end + + def uploads_export_path + @uploads_export_path ||= File.join(@shared.export_path, @relative_export_path) + end + + def each_uploader + avatar_path = @project.avatar&.upload&.path + + if @relative_export_path == 'avatar' + yield(@project.avatar) + else + project_uploads_except_avatar(avatar_path).find_each(batch_size: UPLOADS_BATCH_SIZE) do |upload| + yield(upload.build_uploader) + end + end + end + + def project_uploads_except_avatar(avatar_path) + return @project.uploads unless avatar_path + + @project.uploads.where("path != ?", avatar_path) + end + + def download_and_copy(upload) + secret = upload.try(:secret) || '' + upload_path = File.join(uploads_export_path, secret, upload.filename) + + mkdir_p(File.join(uploads_export_path, secret)) + + File.open(upload_path, 'w') do |file| + # Download (stream) file from the uploader's location + IO.copy_stream(URI.parse(upload.file.url).open, file) + end + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index df19354b76e..25f85936227 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -2,13 +2,30 @@ module Gitlab module ImportExport class UploadsRestorer < UploadsSaver def restore - return true unless File.directory?(uploads_export_path) + if Gitlab::ImportExport.object_storage? + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).restore + elsif File.directory?(uploads_export_path) + copy_files(uploads_export_path, uploads_path) - copy_files(uploads_export_path, uploads_path) + true + else + true # Proceed without uploads + end rescue => e @shared.error(e) false end + + def uploads_path + FileUploader.absolute_base_dir(@project) + end + + def uploads_export_path + @uploads_export_path ||= File.join(@shared.export_path, 'uploads') + end end end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 2f08dda55fd..b3f17af5661 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -9,21 +9,14 @@ module Gitlab end def save - return true unless File.directory?(uploads_path) - - copy_files(uploads_path, uploads_export_path) + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).save rescue => e @shared.error(e) false end - - def uploads_path - FileUploader.absolute_base_dir(@project) - end - - def uploads_export_path - File.join(@shared.export_path, 'uploads') - end end end end |