WIP Attachments migration

4 files changed, 172 insertions, 1 deletions

diff --git a/app/services/projects/hashed_storage/migrate_attachments_service.rb b/app/services/projects/hashed_storage/migrate_attachments_service.rb
new file mode 100644
index 00000000000..58a47da2fcb
--- /dev/null
+++ b/app/services/projects/hashed_storage/migrate_attachments_service.rb
@@ -0,0 +1,54 @@
+module Projects
+  module HashedStorage
+    class MigrateAttachmentsService < BaseService
+      attr_reader :logger
+
+      BATCH_SIZE = 500
+
+      def initialize(project, logger = nil)
+        @project = project
+        @logger = logger || Rails.logger
+      end
+
+      def execute
+        project_before_migration = project.dup
+        project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments]
+
+        project.uploads.find_each(batch_size: BATCH_SIZE) do |upload|
+          old_path = attachments_path(project_before_migration, upload)
+          new_path = attachments_path(project, upload)
+          move_attachment(old_path, new_path)
+        end
+
+        project.save!
+      end
+
+      private
+
+      def attachments_path(project, upload)
+        File.join(
+          FileUploader.dynamic_path_segment(project),
+          upload.path
+        )
+      end
+
+      def move_attachment(old_path, new_path)
+        unless File.file?(old_path)
+          logger.error("Failed to migrate attachment from '#{old_path}' to '#{new_path}', source file doesn't exist (PROJECT_ID=#{project.id})")
+          return
+        end
+
+        # Create attachments folder if doesn't exist yet
+        FileUtils.mkdir_p(File.dirname(new_path)) unless Dir.exist?(File.dirname(new_path))
+
+        if File.file?(new_path)
+          logger.info("Skipped attachment migration from '#{old_path}' to '#{new_path}', target file already exist (PROJECT_ID=#{project.id})")
+          return
+        end
+
+        FileUtils.mv(old_path, new_path)
+        logger.info("Migrated project attachment from '#{old_path}' to '#{new_path}' (PROJECT_ID=#{project.id})")
+      end
+    end
+  end
+end
diff --git a/app/services/projects/hashed_storage_migration_service.rb b/app/services/projects/hashed_storage_migration_service.rb
index b61f71cf9a0..662702c1db5 100644
--- a/app/services/projects/hashed_storage_migration_service.rb
+++ b/app/services/projects/hashed_storage_migration_service.rb
@@ -1,7 +1,7 @@
 module Projects
   class HashedStorageMigrationService < BaseService
     attr_reader :logger
-    
+
     def initialize(project, logger = nil)
       @project = project
       @logger = logger || Rails.logger
@@ -12,6 +12,11 @@ module Projects
       unless project.hashed_storage?(:repository)
         return unless HashedStorage::MigrateRepositoryService.new(project, logger).execute
       end
+
+      # Migrate attachments from Legacy to Hashed Storage
+      unless project.hashed_storage?(:attachments)
+        HashedStorage::MigrateAttachmentsService.new(project, logger).execute
+      end
     end
   end
 end
diff --git a/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb b/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb
new file mode 100644
index 00000000000..81f05074261
--- /dev/null
+++ b/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb
@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+describe Projects::HashedStorage::MigrateAttachmentsService do
+  subject(:service) { described_class.new(project) }
+  let(:project) { create(:project) }
+  let(:legacy_storage) { Storage::LegacyProject.new(project) }
+  let(:hashed_storage) { Storage::HashedProject.new(project) }
+
+  let!(:upload) { Upload.find_by(path: file_uploader.relative_path) }
+  let(:file_uploader) { build(:file_uploader, project: project) }
+  let(:old_path) { attachments_path(legacy_storage, upload) }
+  let(:new_path) { attachments_path(hashed_storage, upload) }
+
+  let(:other_file_uploader) { build(:file_uploader, project: project) }
+  let(:other_old_path) { attachments_path(legacy_storage, other_upload) }
+  let(:other_new_path) { attachments_path(hashed_storage, other_upload) }
+
+  context '#execute' do
+    context 'when succeeds' do
+      it 'moves attachments to hashed storage layout' do
+        expect(File.file?(old_path)).to be_truthy
+        expect(File.file?(new_path)).to be_falsey
+
+        service.execute
+
+        expect(File.file?(old_path)).to be_falsey
+        expect(File.file?(new_path)).to be_truthy
+      end
+    end
+
+    context 'when original file does not exist anymore' do
+      let!(:other_upload) { Upload.find_by(path: other_file_uploader.relative_path) }
+
+      before do
+        File.unlink(old_path)
+      end
+
+      it 'skips moving the file and goes to next' do
+        expect(FileUtils).not_to receive(:mv).with(old_path, new_path)
+        expect(FileUtils).to receive(:mv).with(other_old_path, other_new_path).and_call_original
+
+        service.execute
+
+        expect(File.file?(new_path)).to be_falsey
+        expect(File.file?(other_new_path)).to be_truthy
+      end
+    end
+
+    context 'when target file already exists' do
+      let!(:other_upload) { Upload.find_by(path: other_file_uploader.relative_path) }
+
+      before do
+        FileUtils.mkdir_p(File.dirname(new_path))
+        FileUtils.touch(new_path)
+      end
+
+      it 'skips moving the file and goes to next' do
+        expect(FileUtils).not_to receive(:mv).with(old_path, new_path)
+        expect(FileUtils).to receive(:mv).with(other_old_path, other_new_path).and_call_original
+        expect(File.file?(new_path)).to be_truthy
+
+        service.execute
+
+        expect(File.file?(old_path)).to be_truthy
+      end
+    end
+  end
+
+  def attachments_path(storage, upload)
+    File.join(CarrierWave.root, FileUploader.base_dir, storage.disk_path, upload.path)
+  end
+end
diff --git a/spec/services/projects/hashed_storage_migration_service_spec.rb b/spec/services/projects/hashed_storage_migration_service_spec.rb
new file mode 100644
index 00000000000..28b6daf217e
--- /dev/null
+++ b/spec/services/projects/hashed_storage_migration_service_spec.rb
@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe Projects::HashedStorageMigrationService do
+  let(:project) { create(:project, :empty_repo, :wiki_repo) }
+  subject(:service) { described_class.new(project) }
+
+  describe '#execute' do
+    context 'repository migration' do
+      it 'delegates migration to Projects::HashedStorage::MigrateRepositoryService' do
+        expect(Projects::HashedStorage::MigrateRepositoryService).to receive(:new).with(project, subject.logger).and_call_original
+        expect_any_instance_of(Projects::HashedStorage::MigrateRepositoryService).to receive(:execute)
+
+        service.execute
+      end
+
+      it 'does not delegate migration if repository is already migrated' do
+        project.storage_version = ::Project::LATEST_STORAGE_VERSION
+        expect_any_instance_of(Projects::HashedStorage::MigrateRepositoryService).not_to receive(:execute)
+
+        service.execute
+      end
+    end
+
+    context 'attachments migration' do
+      it 'delegates migration to Projects::HashedStorage::MigrateRepositoryService' do
+        expect(Projects::HashedStorage::MigrateAttachmentsService).to receive(:new).with(project, subject.logger).and_call_original
+        expect_any_instance_of(Projects::HashedStorage::MigrateAttachmentsService).to receive(:execute)
+
+        service.execute
+      end
+
+      it 'does not delegate migration if attachments are already migrated' do
+        project.storage_version = ::Project::LATEST_STORAGE_VERSION
+        expect_any_instance_of(Projects::HashedStorage::MigrateAttachmentsService).not_to receive(:execute)
+
+        service.execute
+      end
+    end
+  end
+end