diff options
4 files changed, 172 insertions, 1 deletions
diff --git a/app/services/projects/hashed_storage/migrate_attachments_service.rb b/app/services/projects/hashed_storage/migrate_attachments_service.rb new file mode 100644 index 00000000000..58a47da2fcb --- /dev/null +++ b/app/services/projects/hashed_storage/migrate_attachments_service.rb @@ -0,0 +1,54 @@ +module Projects + module HashedStorage + class MigrateAttachmentsService < BaseService + attr_reader :logger + + BATCH_SIZE = 500 + + def initialize(project, logger = nil) + @project = project + @logger = logger || Rails.logger + end + + def execute + project_before_migration = project.dup + project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments] + + project.uploads.find_each(batch_size: BATCH_SIZE) do |upload| + old_path = attachments_path(project_before_migration, upload) + new_path = attachments_path(project, upload) + move_attachment(old_path, new_path) + end + + project.save! + end + + private + + def attachments_path(project, upload) + File.join( + FileUploader.dynamic_path_segment(project), + upload.path + ) + end + + def move_attachment(old_path, new_path) + unless File.file?(old_path) + logger.error("Failed to migrate attachment from '#{old_path}' to '#{new_path}', source file doesn't exist (PROJECT_ID=#{project.id})") + return + end + + # Create attachments folder if doesn't exist yet + FileUtils.mkdir_p(File.dirname(new_path)) unless Dir.exist?(File.dirname(new_path)) + + if File.file?(new_path) + logger.info("Skipped attachment migration from '#{old_path}' to '#{new_path}', target file already exist (PROJECT_ID=#{project.id})") + return + end + + FileUtils.mv(old_path, new_path) + logger.info("Migrated project attachment from '#{old_path}' to '#{new_path}' (PROJECT_ID=#{project.id})") + end + end + end +end diff --git a/app/services/projects/hashed_storage_migration_service.rb b/app/services/projects/hashed_storage_migration_service.rb index b61f71cf9a0..662702c1db5 100644 --- a/app/services/projects/hashed_storage_migration_service.rb +++ b/app/services/projects/hashed_storage_migration_service.rb @@ -1,7 +1,7 @@ module Projects class HashedStorageMigrationService < BaseService attr_reader :logger - + def initialize(project, logger = nil) @project = project @logger = logger || Rails.logger @@ -12,6 +12,11 @@ module Projects unless project.hashed_storage?(:repository) return unless HashedStorage::MigrateRepositoryService.new(project, logger).execute end + + # Migrate attachments from Legacy to Hashed Storage + unless project.hashed_storage?(:attachments) + HashedStorage::MigrateAttachmentsService.new(project, logger).execute + end end end end diff --git a/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb b/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb new file mode 100644 index 00000000000..81f05074261 --- /dev/null +++ b/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb @@ -0,0 +1,72 @@ +require 'spec_helper' + +describe Projects::HashedStorage::MigrateAttachmentsService do + subject(:service) { described_class.new(project) } + let(:project) { create(:project) } + let(:legacy_storage) { Storage::LegacyProject.new(project) } + let(:hashed_storage) { Storage::HashedProject.new(project) } + + let!(:upload) { Upload.find_by(path: file_uploader.relative_path) } + let(:file_uploader) { build(:file_uploader, project: project) } + let(:old_path) { attachments_path(legacy_storage, upload) } + let(:new_path) { attachments_path(hashed_storage, upload) } + + let(:other_file_uploader) { build(:file_uploader, project: project) } + let(:other_old_path) { attachments_path(legacy_storage, other_upload) } + let(:other_new_path) { attachments_path(hashed_storage, other_upload) } + + context '#execute' do + context 'when succeeds' do + it 'moves attachments to hashed storage layout' do + expect(File.file?(old_path)).to be_truthy + expect(File.file?(new_path)).to be_falsey + + service.execute + + expect(File.file?(old_path)).to be_falsey + expect(File.file?(new_path)).to be_truthy + end + end + + context 'when original file does not exist anymore' do + let!(:other_upload) { Upload.find_by(path: other_file_uploader.relative_path) } + + before do + File.unlink(old_path) + end + + it 'skips moving the file and goes to next' do + expect(FileUtils).not_to receive(:mv).with(old_path, new_path) + expect(FileUtils).to receive(:mv).with(other_old_path, other_new_path).and_call_original + + service.execute + + expect(File.file?(new_path)).to be_falsey + expect(File.file?(other_new_path)).to be_truthy + end + end + + context 'when target file already exists' do + let!(:other_upload) { Upload.find_by(path: other_file_uploader.relative_path) } + + before do + FileUtils.mkdir_p(File.dirname(new_path)) + FileUtils.touch(new_path) + end + + it 'skips moving the file and goes to next' do + expect(FileUtils).not_to receive(:mv).with(old_path, new_path) + expect(FileUtils).to receive(:mv).with(other_old_path, other_new_path).and_call_original + expect(File.file?(new_path)).to be_truthy + + service.execute + + expect(File.file?(old_path)).to be_truthy + end + end + end + + def attachments_path(storage, upload) + File.join(CarrierWave.root, FileUploader.base_dir, storage.disk_path, upload.path) + end +end diff --git a/spec/services/projects/hashed_storage_migration_service_spec.rb b/spec/services/projects/hashed_storage_migration_service_spec.rb new file mode 100644 index 00000000000..28b6daf217e --- /dev/null +++ b/spec/services/projects/hashed_storage_migration_service_spec.rb @@ -0,0 +1,40 @@ +require 'spec_helper' + +describe Projects::HashedStorageMigrationService do + let(:project) { create(:project, :empty_repo, :wiki_repo) } + subject(:service) { described_class.new(project) } + + describe '#execute' do + context 'repository migration' do + it 'delegates migration to Projects::HashedStorage::MigrateRepositoryService' do + expect(Projects::HashedStorage::MigrateRepositoryService).to receive(:new).with(project, subject.logger).and_call_original + expect_any_instance_of(Projects::HashedStorage::MigrateRepositoryService).to receive(:execute) + + service.execute + end + + it 'does not delegate migration if repository is already migrated' do + project.storage_version = ::Project::LATEST_STORAGE_VERSION + expect_any_instance_of(Projects::HashedStorage::MigrateRepositoryService).not_to receive(:execute) + + service.execute + end + end + + context 'attachments migration' do + it 'delegates migration to Projects::HashedStorage::MigrateRepositoryService' do + expect(Projects::HashedStorage::MigrateAttachmentsService).to receive(:new).with(project, subject.logger).and_call_original + expect_any_instance_of(Projects::HashedStorage::MigrateAttachmentsService).to receive(:execute) + + service.execute + end + + it 'does not delegate migration if attachments are already migrated' do + project.storage_version = ::Project::LATEST_STORAGE_VERSION + expect_any_instance_of(Projects::HashedStorage::MigrateAttachmentsService).not_to receive(:execute) + + service.execute + end + end + end +end |