Added Rollbacker workers and support on the rake task

Rollback is done similar to Migration for the Hashed Storage.
It also shares the same ExclusiveLease key to prevent both happening
at the same time.

All Hashed Storage related workers now share the same queue namespace
which allows for assigning dedicated workers easily.

5 files changed, 184 insertions, 7 deletions

diff --git a/spec/lib/gitlab/hashed_storage/migrator_spec.rb b/spec/lib/gitlab/hashed_storage/migrator_spec.rb
index 000913f32bb..6154b3e2f76 100644
--- a/spec/lib/gitlab/hashed_storage/migrator_spec.rb
+++ b/spec/lib/gitlab/hashed_storage/migrator_spec.rb
@@ -1,16 +1,24 @@
 require 'spec_helper'
 
 describe Gitlab::HashedStorage::Migrator do
-  describe '#bulk_schedule' do
-    it 'schedules job to StorageMigratorWorker' do
+  describe '#bulk_schedule_migration' do
+    it 'schedules job to HashedStorage::MigratorWorker' do
       Sidekiq::Testing.fake! do
-        expect { subject.bulk_schedule(start: 1, finish: 5) }.to change(HashedStorage::MigratorWorker.jobs, :size).by(1)
+        expect { subject.bulk_schedule_migration(start: 1, finish: 5) }.to change(HashedStorage::MigratorWorker.jobs, :size).by(1)
+      end
+    end
+  end
+
+  describe '#bulk_schedule_rollback' do
+    it 'schedules job to HashedStorage::RollbackerWorker' do
+      Sidekiq::Testing.fake! do
+        expect { subject.bulk_schedule_rollback(start: 1, finish: 5) }.to change(HashedStorage::RollbackerWorker.jobs, :size).by(1)
       end
     end
   end
 
   describe '#bulk_migrate' do
-    let(:projects) { create_list(:project, 2, :legacy_storage) }
+    let(:projects) { create_list(:project, 2, :legacy_storage, :empty_repo) }
     let(:ids) { projects.map(&:id) }
 
     it 'enqueue jobs to HashedStorage::ProjectMigrateWorker' do
@@ -32,13 +40,53 @@ describe Gitlab::HashedStorage::Migrator do
       subject.bulk_migrate(start: ids.min, finish: ids.max)
     end
 
-    it 'has migrated projects set as writable' do
+    it 'has all projects migrated and set as writable' do
       perform_enqueued_jobs do
         subject.bulk_migrate(start: ids.min, finish: ids.max)
       end
 
       projects.each do |project|
-        expect(project.reload.repository_read_only?).to be_falsey
+        project.reload
+
+        expect(project.hashed_storage?(:repository)).to be_truthy
+        expect(project.repository_read_only?).to be_falsey
+      end
+    end
+  end
+
+  describe '#bulk_rollback' do
+    let(:projects) { create_list(:project, 2, :empty_repo) }
+    let(:ids) { projects.map(&:id) }
+
+    it 'enqueue jobs to HashedStorage::ProjectRollbackWorker' do
+      Sidekiq::Testing.fake! do
+        expect { subject.bulk_rollback(start: ids.min, finish: ids.max) }.to change(HashedStorage::ProjectRollbackWorker.jobs, :size).by(2)
+      end
+    end
+
+    it 'rescues and log exceptions' do
+      allow_any_instance_of(Project).to receive(:rollback_to_legacy_storage!).and_raise(StandardError)
+      expect { subject.bulk_rollback(start: ids.min, finish: ids.max) }.not_to raise_error
+    end
+
+    it 'delegates each project in specified range to #rollback' do
+      projects.each do |project|
+        expect(subject).to receive(:rollback).with(project)
+      end
+
+      subject.bulk_rollback(start: ids.min, finish: ids.max)
+    end
+
+    it 'has all projects rolledback and set as writable' do
+      perform_enqueued_jobs do
+        subject.bulk_rollback(start: ids.min, finish: ids.max)
+      end
+
+      projects.each do |project|
+        project.reload
+
+        expect(project.legacy_storage?).to be_truthy
+        expect(project.repository_read_only?).to be_falsey
       end
     end
   end
diff --git a/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb b/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb
index 61dbb57ec08..d51988c6f42 100644
--- a/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb
+++ b/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb
@@ -86,6 +86,8 @@ describe Projects::HashedStorage::MigrateAttachmentsService do
 
   context '#new_disk_path' do
     it 'returns new disk_path for project' do
+      service.execute
+
       expect(service.new_disk_path).to eq(project.disk_path)
     end
   end
diff --git a/spec/services/projects/hashed_storage/rollback_attachments_service_spec.rb b/spec/services/projects/hashed_storage/rollback_attachments_service_spec.rb
new file mode 100644
index 00000000000..9e18428f412
--- /dev/null
+++ b/spec/services/projects/hashed_storage/rollback_attachments_service_spec.rb
@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Projects::HashedStorage::RollbackAttachmentsService do
+  subject(:service) { described_class.new(project, logger: nil) }
+
+  let(:project) { create(:project, :repository, skip_disk_validation: true) }
+  let(:legacy_storage) { Storage::LegacyProject.new(project) }
+  let(:hashed_storage) { Storage::HashedProject.new(project) }
+
+  let!(:upload) { Upload.find_by(path: file_uploader.upload_path) }
+  let(:file_uploader) { build(:file_uploader, project: project) }
+  let(:old_disk_path) { File.join(base_path(hashed_storage), upload.path) }
+  let(:new_disk_path) { File.join(base_path(legacy_storage), upload.path) }
+
+  context '#execute' do
+    context 'when succeeds' do
+      it 'moves attachments to legacy storage layout' do
+        expect(File.file?(old_disk_path)).to be_truthy
+        expect(File.file?(new_disk_path)).to be_falsey
+        expect(File.exist?(base_path(hashed_storage))).to be_truthy
+        expect(File.exist?(base_path(legacy_storage))).to be_falsey
+        expect(FileUtils).to receive(:mv).with(base_path(hashed_storage), base_path(legacy_storage)).and_call_original
+
+        service.execute
+
+        expect(File.exist?(base_path(legacy_storage))).to be_truthy
+        expect(File.exist?(base_path(hashed_storage))).to be_falsey
+        expect(File.file?(old_disk_path)).to be_falsey
+        expect(File.file?(new_disk_path)).to be_truthy
+      end
+
+      it 'returns true' do
+        expect(service.execute).to be_truthy
+      end
+
+      it 'sets skipped to false' do
+        service.execute
+
+        expect(service.skipped?).to be_falsey
+      end
+    end
+
+    context 'when original folder does not exist anymore' do
+      before do
+        FileUtils.rm_rf(base_path(hashed_storage))
+      end
+
+      it 'skips moving folders and go to next' do
+        expect(FileUtils).not_to receive(:mv).with(base_path(hashed_storage), base_path(legacy_storage))
+
+        service.execute
+
+        expect(File.exist?(base_path(legacy_storage))).to be_falsey
+        expect(File.file?(new_disk_path)).to be_falsey
+      end
+
+      it 'returns true' do
+        expect(service.execute).to be_truthy
+      end
+
+      it 'sets skipped to true' do
+        service.execute
+
+        expect(service.skipped?).to be_truthy
+      end
+    end
+
+    context 'when target folder already exists' do
+      before do
+        FileUtils.mkdir_p(base_path(legacy_storage))
+      end
+
+      it 'raises AttachmentRollbackError' do
+        expect(FileUtils).not_to receive(:mv).with(base_path(legacy_storage), base_path(hashed_storage))
+
+        expect { service.execute }.to raise_error(Projects::HashedStorage::AttachmentRollbackError)
+      end
+    end
+  end
+
+  context '#old_disk_path' do
+    it 'returns old disk_path for project' do
+      expect(service.old_disk_path).to eq(project.disk_path)
+    end
+  end
+
+  context '#new_disk_path' do
+    it 'returns new disk_path for project' do
+      service.execute
+
+      expect(service.new_disk_path).to eq(project.full_path)
+    end
+  end
+
+  def base_path(storage)
+    File.join(FileUploader.root, storage.disk_path)
+  end
+end
diff --git a/spec/services/projects/hashed_storage/rollback_repository_service_spec.rb b/spec/services/projects/hashed_storage/rollback_repository_service_spec.rb
index c9d0a085d21..7c3a243b3ca 100644
--- a/spec/services/projects/hashed_storage/rollback_repository_service_spec.rb
+++ b/spec/services/projects/hashed_storage/rollback_repository_service_spec.rb
@@ -66,7 +66,7 @@ describe Projects::HashedStorage::RollbackRepositoryService, :clean_gitlab_redis
     end
 
     context 'when one move fails' do
-      it 'rollsback repositories to original name' do
+      it 'rolls repositories back to original name' do
         allow(service).to receive(:move_repository).and_call_original
         allow(service).to receive(:move_repository).with(old_disk_path, new_disk_path).once { false } # will disable first move only
 
diff --git a/spec/workers/hashed_storage/rollbacker_worker_spec.rb b/spec/workers/hashed_storage/rollbacker_worker_spec.rb
new file mode 100644
index 00000000000..4055f380978
--- /dev/null
+++ b/spec/workers/hashed_storage/rollbacker_worker_spec.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe HashedStorage::RollbackerWorker do
+  subject(:worker) { described_class.new }
+  let(:projects) { create_list(:project, 2, :empty_repo) }
+  let(:ids) { projects.map(&:id) }
+
+  describe '#perform' do
+    it 'delegates to MigratorService' do
+      expect_any_instance_of(Gitlab::HashedStorage::Migrator).to receive(:bulk_rollback).with(start: 5, finish: 10)
+
+      worker.perform(5, 10)
+    end
+
+    it 'rollsback projects in the specified range' do
+      perform_enqueued_jobs do
+        worker.perform(ids.min, ids.max)
+      end
+
+      projects.each do |project|
+        expect(project.reload.legacy_storage?).to be_truthy
+      end
+    end
+  end
+end