Merge branch 'hashed-storage-attachments-migration-path' into 'master'

Hashed storage attachments migration path

See merge request gitlab-org/gitlab-ce!15352

14 files changed, 523 insertions, 150 deletions

diff --git a/app/models/project.rb b/app/models/project.rb
index e276bd2422d..85d580fe0fa 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -273,8 +273,9 @@ class Project < ActiveRecord::Base
   scope :pending_delete, -> { where(pending_delete: true) }
   scope :without_deleted, -> { where(pending_delete: false) }
 
-  scope :with_hashed_storage, -> { where('storage_version >= 1') }
-  scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) }
+  scope :with_storage_feature, ->(feature) { where('storage_version >= :version', version: HASHED_STORAGE_FEATURES[feature]) }
+  scope :without_storage_feature, ->(feature) { where('storage_version < :version OR storage_version IS NULL', version: HASHED_STORAGE_FEATURES[feature]) }
+  scope :with_unmigrated_storage, -> { where('storage_version < :version OR storage_version IS NULL', version: LATEST_STORAGE_VERSION) }
 
   scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) }
   scope :sorted_by_stars, -> { reorder('projects.star_count DESC') }
diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb
index f025f40994e..fae1b64961a 100644
--- a/app/models/storage/hashed_project.rb
+++ b/app/models/storage/hashed_project.rb
@@ -4,7 +4,6 @@ module Storage
     delegate :gitlab_shell, :repository_storage_path, to: :project
 
     ROOT_PATH_PREFIX = '@hashed'.freeze
-    STORAGE_VERSION = 1
 
     def initialize(project)
       @project = project
diff --git a/app/services/projects/hashed_storage/migrate_attachments_service.rb b/app/services/projects/hashed_storage/migrate_attachments_service.rb
new file mode 100644
index 00000000000..f8aaec8a9c0
--- /dev/null
+++ b/app/services/projects/hashed_storage/migrate_attachments_service.rb
@@ -0,0 +1,54 @@
+module Projects
+  module HashedStorage
+    AttachmentMigrationError = Class.new(StandardError)
+
+    class MigrateAttachmentsService < BaseService
+      attr_reader :logger, :old_path, :new_path
+
+      def initialize(project, logger = nil)
+        @project = project
+        @logger = logger || Rails.logger
+      end
+
+      def execute
+        @old_path = project.full_path
+        @new_path = project.disk_path
+
+        origin = FileUploader.dynamic_path_segment(project)
+        project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments]
+        target = FileUploader.dynamic_path_segment(project)
+
+        result = move_folder!(origin, target)
+        project.save!
+
+        if result && block_given?
+          yield
+        end
+
+        result
+      end
+
+      private
+
+      def move_folder!(old_path, new_path)
+        unless File.directory?(old_path)
+          logger.info("Skipped attachments migration from '#{old_path}' to '#{new_path}', source path doesn't exist or is not a directory (PROJECT_ID=#{project.id})")
+          return
+        end
+
+        if File.exist?(new_path)
+          logger.error("Cannot migrate attachments from '#{old_path}' to '#{new_path}', target path already exist (PROJECT_ID=#{project.id})")
+          raise AttachmentMigrationError, "Target path '#{new_path}' already exist"
+        end
+
+        # Create hashed storage base path folder
+        FileUtils.mkdir_p(File.dirname(new_path))
+
+        FileUtils.mv(old_path, new_path)
+        logger.info("Migrated project attachments from '#{old_path}' to '#{new_path}' (PROJECT_ID=#{project.id})")
+
+        true
+      end
+    end
+  end
+end
diff --git a/app/services/projects/hashed_storage/migrate_repository_service.rb b/app/services/projects/hashed_storage/migrate_repository_service.rb
new file mode 100644
index 00000000000..7212e7524ab
--- /dev/null
+++ b/app/services/projects/hashed_storage/migrate_repository_service.rb
@@ -0,0 +1,70 @@
+module Projects
+  module HashedStorage
+    class MigrateRepositoryService < BaseService
+      include Gitlab::ShellAdapter
+
+      attr_reader :old_disk_path, :new_disk_path, :old_wiki_disk_path, :old_storage_version, :logger
+
+      def initialize(project, logger = nil)
+        @project = project
+        @logger = logger || Rails.logger
+      end
+
+      def execute
+        @old_disk_path = project.disk_path
+        has_wiki = project.wiki.repository_exists?
+
+        @old_storage_version = project.storage_version
+        project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:repository]
+        project.ensure_storage_path_exists
+
+        @new_disk_path = project.disk_path
+
+        result = move_repository(@old_disk_path, @new_disk_path)
+
+        if has_wiki
+          @old_wiki_disk_path = "#{@old_disk_path}.wiki"
+          result &&= move_repository("#{@old_wiki_disk_path}", "#{@new_disk_path}.wiki")
+        end
+
+        unless result
+          rollback_folder_move
+          project.storage_version = nil
+        end
+
+        project.repository_read_only = false
+        project.save!
+
+        if result && block_given?
+          yield
+        end
+
+        result
+      end
+
+      private
+
+      def move_repository(from_name, to_name)
+        from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git")
+        to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git")
+
+        # If we don't find the repository on either original or target we should log that as it could be an issue if the
+        # project was not originally empty.
+        if !from_exists && !to_exists
+          logger.warn "Can't find a repository on either source or target paths for #{project.full_path} (ID=#{project.id}) ..."
+          return false
+        elsif !from_exists
+          # Repository have been moved already.
+          return true
+        end
+
+        gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
+      end
+
+      def rollback_folder_move
+        move_repository(@new_disk_path, @old_disk_path)
+        move_repository("#{@new_disk_path}.wiki", "#{@old_disk_path}.wiki")
+      end
+    end
+  end
+end
diff --git a/app/services/projects/hashed_storage_migration_service.rb b/app/services/projects/hashed_storage_migration_service.rb
index f5945f3b87f..662702c1db5 100644
--- a/app/services/projects/hashed_storage_migration_service.rb
+++ b/app/services/projects/hashed_storage_migration_service.rb
@@ -1,68 +1,22 @@
 module Projects
   class HashedStorageMigrationService < BaseService
-    include Gitlab::ShellAdapter
-
-    attr_reader :old_disk_path, :new_disk_path
+    attr_reader :logger
 
     def initialize(project, logger = nil)
       @project = project
-      @logger ||= Rails.logger
+      @logger = logger || Rails.logger
     end
 
     def execute
-      return if project.hashed_storage?(:repository)
-
-      @old_disk_path = project.disk_path
-      has_wiki = project.wiki.repository_exists?
-
-      project.storage_version = Storage::HashedProject::STORAGE_VERSION
-      project.ensure_storage_path_exists
-
-      @new_disk_path = project.disk_path
-
-      result = move_repository(@old_disk_path, @new_disk_path)
-
-      if has_wiki
-        result &&= move_repository("#{@old_disk_path}.wiki", "#{@new_disk_path}.wiki")
-      end
-
-      unless result
-        rollback_folder_move
-        return
+      # Migrate repository from Legacy to Hashed Storage
+      unless project.hashed_storage?(:repository)
+        return unless HashedStorage::MigrateRepositoryService.new(project, logger).execute
       end
 
-      project.repository_read_only = false
-      project.save!
-
-      block_given? ? yield : result
-    end
-
-    private
-
-    def move_repository(from_name, to_name)
-      from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git")
-      to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git")
-
-      # If we don't find the repository on either original or target we should log that as it could be an issue if the
-      # project was not originally empty.
-      if !from_exists && !to_exists
-        logger.warn "Can't find a repository on either source or target paths for #{project.full_path} (ID=#{project.id}) ..."
-        return false
-      elsif !from_exists
-        # Repository have been moved already.
-        return true
+      # Migrate attachments from Legacy to Hashed Storage
+      unless project.hashed_storage?(:attachments)
+        HashedStorage::MigrateAttachmentsService.new(project, logger).execute
       end
-
-      gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
-    end
-
-    def rollback_folder_move
-      move_repository(@new_disk_path, @old_disk_path)
-      move_repository("#{@new_disk_path}.wiki", "#{@old_disk_path}.wiki")
-    end
-
-    def logger
-      @logger
     end
   end
 end
diff --git a/app/uploaders/file_uploader.rb b/app/uploaders/file_uploader.rb
index f4a5cf75018..71658df5b41 100644
--- a/app/uploaders/file_uploader.rb
+++ b/app/uploaders/file_uploader.rb
@@ -31,12 +31,19 @@ class FileUploader < GitlabUploader
   # Returns a String without a trailing slash
   def self.dynamic_path_segment(project)
     if project.hashed_storage?(:attachments)
-      File.join(CarrierWave.root, base_dir, project.disk_path)
+      dynamic_path_builder(project.disk_path)
     else
-      File.join(CarrierWave.root, base_dir, project.full_path)
+      dynamic_path_builder(project.full_path)
     end
   end
 
+  # Auxiliary method to build dynamic path segment when not using a project model
+  #
+  # Prefer to use the `.dynamic_path_segment` as it includes Hashed Storage specific logic
+  def self.dynamic_path_builder(path)
+    File.join(CarrierWave.root, base_dir, path)
+  end
+
   attr_accessor :model
   attr_reader :secret
 
diff --git a/app/workers/project_migrate_hashed_storage_worker.rb b/app/workers/project_migrate_hashed_storage_worker.rb
index ca276d7801c..127aa6b9d7d 100644
--- a/app/workers/project_migrate_hashed_storage_worker.rb
+++ b/app/workers/project_migrate_hashed_storage_worker.rb
@@ -2,10 +2,34 @@ class ProjectMigrateHashedStorageWorker
   include Sidekiq::Worker
   include DedicatedSidekiqQueue
 
+  LEASE_TIMEOUT = 30.seconds.to_i
+
   def perform(project_id)
     project = Project.find_by(id: project_id)
     return if project.nil? || project.pending_delete?
 
-    ::Projects::HashedStorageMigrationService.new(project, logger).execute
+    uuid = lease_for(project_id).try_obtain
+    if uuid
+      ::Projects::HashedStorageMigrationService.new(project, logger).execute
+    else
+      false
+    end
+  rescue => ex
+    cancel_lease_for(project_id, uuid) if uuid
+    raise ex
+  end
+
+  def lease_for(project_id)
+    Gitlab::ExclusiveLease.new(lease_key(project_id), timeout: LEASE_TIMEOUT)
+  end
+
+  private
+
+  def lease_key(project_id)
+    "project_migrate_hashed_storage_worker:#{project_id}"
+  end
+
+  def cancel_lease_for(project_id, uuid)
+    Gitlab::ExclusiveLease.cancel(lease_key(project_id), uuid)
   end
 end
diff --git a/changelogs/unreleased/hashed-storage-attachments-migration-path.yml b/changelogs/unreleased/hashed-storage-attachments-migration-path.yml
new file mode 100644
index 00000000000..32535437046
--- /dev/null
+++ b/changelogs/unreleased/hashed-storage-attachments-migration-path.yml
@@ -0,0 +1,5 @@
+---
+title: Hashed Storage migration script now supports migrating project attachments
+merge_request: 15352
+author:
+type: added
diff --git a/doc/administration/raketasks/storage.md b/doc/administration/raketasks/storage.md
index bac8fa4bd9d..6ec5baeb6e3 100644
--- a/doc/administration/raketasks/storage.md
+++ b/doc/administration/raketasks/storage.md
@@ -1,10 +1,43 @@
 # Repository Storage Rake Tasks
 
 This is a collection of rake tasks you can use to help you list and migrate 
-existing projects from Legacy storage to the new Hashed storage type.
+existing projects and attachments associated with it from Legacy storage to 
+the new Hashed storage type.
 
 You can read more about the storage types [here][storage-types].
 
+## Migrate existing projects to Hashed storage
+
+Before migrating your existing projects, you should 
+[enable hashed storage][storage-migration] for the new projects as well.
+
+This task will schedule all your existing projects and attachments associated with it to be migrated to the 
+**Hashed** storage type:
+
+**Omnibus Installation**
+
+```bash
+gitlab-rake gitlab:storage:migrate_to_hashed
+```
+
+**Source Installation**
+
+```bash
+rake gitlab:storage:migrate_to_hashed
+
+```
+
+You can monitor the progress in the _Admin > Monitoring > Background jobs_ screen.
+There is a specific Queue you can watch to see how long it will take to finish: **project_migrate_hashed_storage**
+
+After it reaches zero, you can confirm every project has been migrated by running the commands bellow. 
+If you find it necessary, you can run this migration script again to schedule missing projects.
+
+Any error or warning will be logged in the sidekiq's log file.
+
+You only need the `gitlab:storage:migrate_to_hashed` rake task to migrate your repositories, but we have additional
+commands below that helps you inspect projects and attachments in both legacy and hashed storage.
+
 ## List projects on Legacy storage
 
 To have a simple summary of projects using **Legacy** storage:
@@ -73,35 +106,73 @@ rake gitlab:storage:list_hashed_projects
 
 ```
 
-## Migrate existing projects to Hashed storage
+## List attachments on Legacy storage
 
-Before migrating your existing projects, you should 
-[enable hashed storage][storage-migration] for the new projects as well.
+To have a simple summary of project attachments using **Legacy** storage:
 
-This task will schedule all your existing projects to be migrated to the 
-**Hashed** storage type:
+**Omnibus Installation**
+
+```bash
+gitlab-rake gitlab:storage:legacy_attachments
+```
+
+**Source Installation**
+
+```bash
+rake gitlab:storage:legacy_attachments
+
+```
+
+------
+
+To list project attachments using **Legacy** storage:
 
 **Omnibus Installation**
 
 ```bash
-gitlab-rake gitlab:storage:migrate_to_hashed
+gitlab-rake gitlab:storage:list_legacy_attachments
 ```
 
 **Source Installation**
 
 ```bash
-rake gitlab:storage:migrate_to_hashed
+rake gitlab:storage:list_legacy_attachments
 
 ```
 
-You can monitor the progress in the _Admin > Monitoring > Background jobs_ screen.
-There is a specific Queue you can watch to see how long it will take to finish: **project_migrate_hashed_storage**
+## List attachments on Hashed storage
 
-After it reaches zero, you can confirm every project has been migrated by running the commands above.
-If you find it necessary, you can run this migration script again to schedule missing projects.
+To have a simple summary of project attachments using **Hashed** storage:
+
+**Omnibus Installation**
+
+```bash
+gitlab-rake gitlab:storage:hashed_attachments
+```
 
-Any error or warning will be logged in the sidekiq log file.
+**Source Installation**
+
+```bash
+rake gitlab:storage:hashed_attachments
+
+```
+
+------
+
+To list project attachments using **Hashed** storage:
+
+**Omnibus Installation**
 
+```bash
+gitlab-rake gitlab:storage:list_hashed_attachments
+```
+
+**Source Installation**
+
+```bash
+rake gitlab:storage:list_hashed_attachments
+
+```
 
 [storage-types]: ../repository_storage_types.md
 [storage-migration]: ../repository_storage_types.md#how-to-migrate-to-hashed-storage
diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake
index e05be4a3405..8ac73bc8ff2 100644
--- a/lib/tasks/gitlab/storage.rake
+++ b/lib/tasks/gitlab/storage.rake
@@ -2,10 +2,10 @@ namespace :gitlab do
   namespace :storage do
     desc 'GitLab | Storage | Migrate existing projects to Hashed Storage'
     task migrate_to_hashed: :environment do
-      legacy_projects_count = Project.with_legacy_storage.count
+      legacy_projects_count = Project.with_unmigrated_storage.count
 
       if legacy_projects_count == 0
-        puts 'There are no projects using legacy storage. Nothing to do!'
+        puts 'There are no projects requiring storage migration. Nothing to do!'
 
         next
       end
@@ -23,22 +23,42 @@ namespace :gitlab do
 
     desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage'
     task legacy_projects: :environment do
-      projects_summary(Project.with_legacy_storage)
+      relation_summary('projects', Project.without_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | List existing projects using Legacy Storage'
     task list_legacy_projects: :environment do
-      projects_list(Project.with_legacy_storage)
+      projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage'
     task hashed_projects: :environment do
-      projects_summary(Project.with_hashed_storage)
+      relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | List existing projects using Hashed Storage'
     task list_hashed_projects: :environment do
-      projects_list(Project.with_hashed_storage)
+      projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository))
+    end
+
+    desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage'
+    task legacy_attachments: :environment do
+      relation_summary('attachments using Legacy Storage', legacy_attachments_relation)
+    end
+
+    desc 'Gitlab | Storage | List existing project attachments using Legacy Storage'
+    task list_legacy_attachments: :environment do
+      attachments_list('attachments using Legacy Storage', legacy_attachments_relation)
+    end
+
+    desc 'Gitlab | Storage | Summary of project attachments using Hashed Storage'
+    task hashed_attachments: :environment do
+      relation_summary('attachments using Hashed Storage', hashed_attachments_relation)
+    end
+
+    desc 'Gitlab | Storage | List existing project attachments using Hashed Storage'
+    task list_hashed_attachments: :environment do
+      attachments_list('attachments using Hashed Storage', hashed_attachments_relation)
     end
 
     def batch_size
@@ -46,29 +66,43 @@ namespace :gitlab do
     end
 
     def project_id_batches(&block)
-      Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
+      Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
         ids = relation.pluck(:id)
 
         yield ids.min, ids.max
       end
     end
 
-    def projects_summary(relation)
-      projects_count = relation.count
-      puts "* Found #{projects_count} projects".color(:green)
+    def legacy_attachments_relation
+      Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments])
+        JOIN projects
+          ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
+      SQL
+    end
+
+    def hashed_attachments_relation
+      Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments])
+        JOIN projects
+        ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
+      SQL
+    end
+
+    def relation_summary(relation_name, relation)
+      relation_count = relation.count
+      puts "* Found #{relation_count} #{relation_name}".color(:green)
 
-      projects_count
+      relation_count
     end
 
-    def projects_list(relation)
-      projects_count = projects_summary(relation)
+    def projects_list(relation_name, relation)
+      relation_count = relation_summary(relation_name, relation)
 
       projects = relation.with_route
       limit = ENV.fetch('LIMIT', 500).to_i
 
-      return unless projects_count > 0
+      return unless relation_count > 0
 
-      puts "  ! Displaying first #{limit} projects..." if projects_count > limit
+      puts "  ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
 
       counter = 0
       projects.find_in_batches(batch_size: batch_size) do |batch|
@@ -81,5 +115,26 @@ namespace :gitlab do
         end
       end
     end
+
+    def attachments_list(relation_name, relation)
+      relation_count = relation_summary(relation_name, relation)
+
+      limit = ENV.fetch('LIMIT', 500).to_i
+
+      return unless relation_count > 0
+
+      puts "  ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
+
+      counter = 0
+      relation.find_in_batches(batch_size: batch_size) do |batch|
+        batch.each do |upload|
+          counter += 1
+
+          puts "  - #{upload.path} (id: #{upload.id})".color(:red)
+
+          return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator
+        end
+      end
+    end
   end
 end
diff --git a/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb b/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb
new file mode 100644
index 00000000000..50e59954f73
--- /dev/null
+++ b/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb
@@ -0,0 +1,63 @@
+require 'spec_helper'
+
+describe Projects::HashedStorage::MigrateAttachmentsService do
+  subject(:service) { described_class.new(project) }
+  let(:project) { create(:project) }
+  let(:legacy_storage) { Storage::LegacyProject.new(project) }
+  let(:hashed_storage) { Storage::HashedProject.new(project) }
+
+  let!(:upload) { Upload.find_by(path: file_uploader.relative_path) }
+  let(:file_uploader) { build(:file_uploader, project: project) }
+  let(:old_path) { File.join(base_path(legacy_storage), upload.path) }
+  let(:new_path) { File.join(base_path(hashed_storage), upload.path) }
+
+  context '#execute' do
+    context 'when succeeds' do
+      it 'moves attachments to hashed storage layout' do
+        expect(File.file?(old_path)).to be_truthy
+        expect(File.file?(new_path)).to be_falsey
+        expect(File.exist?(base_path(legacy_storage))).to be_truthy
+        expect(File.exist?(base_path(hashed_storage))).to be_falsey
+        expect(FileUtils).to receive(:mv).with(base_path(legacy_storage), base_path(hashed_storage)).and_call_original
+
+        service.execute
+
+        expect(File.exist?(base_path(hashed_storage))).to be_truthy
+        expect(File.exist?(base_path(legacy_storage))).to be_falsey
+        expect(File.file?(old_path)).to be_falsey
+        expect(File.file?(new_path)).to be_truthy
+      end
+    end
+
+    context 'when original folder does not exist anymore' do
+      before do
+        FileUtils.rm_rf(base_path(legacy_storage))
+      end
+
+      it 'skips moving folders and go to next' do
+        expect(FileUtils).not_to receive(:mv).with(base_path(legacy_storage), base_path(hashed_storage))
+
+        service.execute
+
+        expect(File.exist?(base_path(hashed_storage))).to be_falsey
+        expect(File.file?(new_path)).to be_falsey
+      end
+    end
+
+    context 'when target folder already exists' do
+      before do
+        FileUtils.mkdir_p(base_path(hashed_storage))
+      end
+
+      it 'raises AttachmentMigrationError' do
+        expect(FileUtils).not_to receive(:mv).with(base_path(legacy_storage), base_path(hashed_storage))
+
+        expect { service.execute }.to raise_error(Projects::HashedStorage::AttachmentMigrationError)
+      end
+    end
+  end
+
+  def base_path(storage)
+    FileUploader.dynamic_path_builder(storage.disk_path)
+  end
+end
diff --git a/spec/services/projects/hashed_storage/migrate_repository_service_spec.rb b/spec/services/projects/hashed_storage/migrate_repository_service_spec.rb
new file mode 100644
index 00000000000..3a3e47fd9c0
--- /dev/null
+++ b/spec/services/projects/hashed_storage/migrate_repository_service_spec.rb
@@ -0,0 +1,76 @@
+require 'spec_helper'
+
+describe Projects::HashedStorage::MigrateRepositoryService do
+  let(:gitlab_shell) { Gitlab::Shell.new }
+  let(:project) { create(:project, :empty_repo, :wiki_repo) }
+  let(:service) { described_class.new(project) }
+  let(:legacy_storage) { Storage::LegacyProject.new(project) }
+  let(:hashed_storage) { Storage::HashedProject.new(project) }
+
+  describe '#execute' do
+    before do
+      allow(service).to receive(:gitlab_shell) { gitlab_shell }
+    end
+
+    context 'when succeeds' do
+      it 'renames project and wiki repositories' do
+        service.execute
+
+        expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.git")).to be_truthy
+        expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.wiki.git")).to be_truthy
+      end
+
+      it 'updates project to be hashed and not read-only' do
+        service.execute
+
+        expect(project.hashed_storage?(:repository)).to be_truthy
+        expect(project.repository_read_only).to be_falsey
+      end
+
+      it 'move operation is called for both repositories' do
+        expect_move_repository(project.disk_path, hashed_storage.disk_path)
+        expect_move_repository("#{project.disk_path}.wiki", "#{hashed_storage.disk_path}.wiki")
+
+        service.execute
+      end
+    end
+
+    context 'when one move fails' do
+      it 'rollsback repositories to original name' do
+        from_name = project.disk_path
+        to_name = hashed_storage.disk_path
+        allow(service).to receive(:move_repository).and_call_original
+        allow(service).to receive(:move_repository).with(from_name, to_name).once { false } # will disable first move only
+
+        expect(service).to receive(:rollback_folder_move).and_call_original
+
+        service.execute
+
+        expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.git")).to be_falsey
+        expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.wiki.git")).to be_falsey
+        expect(project.repository_read_only?).to be_falsey
+      end
+
+      context 'when rollback fails' do
+        let(:from_name) { legacy_storage.disk_path }
+        let(:to_name) { hashed_storage.disk_path }
+
+        before do
+          hashed_storage.ensure_storage_path_exists
+          gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
+        end
+
+        it 'does not try to move nil repository over hashed' do
+          expect(gitlab_shell).not_to receive(:mv_repository).with(project.repository_storage_path, from_name, to_name)
+          expect_move_repository("#{project.disk_path}.wiki", "#{hashed_storage.disk_path}.wiki")
+
+          service.execute
+        end
+      end
+    end
+
+    def expect_move_repository(from_name, to_name)
+      expect(gitlab_shell).to receive(:mv_repository).with(project.repository_storage_path, from_name, to_name).and_call_original
+    end
+  end
+end
diff --git a/spec/services/projects/hashed_storage_migration_service_spec.rb b/spec/services/projects/hashed_storage_migration_service_spec.rb
index b71b47c59b6..466f0b5d7c2 100644
--- a/spec/services/projects/hashed_storage_migration_service_spec.rb
+++ b/spec/services/projects/hashed_storage_migration_service_spec.rb
@@ -1,74 +1,44 @@
 require 'spec_helper'
 
 describe Projects::HashedStorageMigrationService do
-  let(:gitlab_shell) { Gitlab::Shell.new }
   let(:project) { create(:project, :empty_repo, :wiki_repo) }
-  let(:service) { described_class.new(project) }
-  let(:legacy_storage) { Storage::LegacyProject.new(project) }
-  let(:hashed_storage) { Storage::HashedProject.new(project) }
+  subject(:service) { described_class.new(project) }
 
   describe '#execute' do
-    before do
-      allow(service).to receive(:gitlab_shell) { gitlab_shell }
-    end
-
-    context 'when succeeds' do
-      it 'renames project and wiki repositories' do
-        service.execute
+    context 'repository migration' do
+      let(:repository_service) { Projects::HashedStorage::MigrateRepositoryService.new(project, subject.logger) }
 
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.git")).to be_truthy
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.wiki.git")).to be_truthy
-      end
+      it 'delegates migration to Projects::HashedStorage::MigrateRepositoryService' do
+        expect(Projects::HashedStorage::MigrateRepositoryService).to receive(:new).with(project, subject.logger).and_return(repository_service)
+        expect(repository_service).to receive(:execute)
 
-      it 'updates project to be hashed and not read-only' do
         service.execute
-
-        expect(project.hashed_storage?(:repository)).to be_truthy
-        expect(project.repository_read_only).to be_falsey
       end
 
-      it 'move operation is called for both repositories' do
-        expect_move_repository(project.disk_path, hashed_storage.disk_path)
-        expect_move_repository("#{project.disk_path}.wiki", "#{hashed_storage.disk_path}.wiki")
+      it 'does not delegate migration if repository is already migrated' do
+        project.storage_version = ::Project::LATEST_STORAGE_VERSION
+        expect(Projects::HashedStorage::MigrateRepositoryService).not_to receive(:new)
 
         service.execute
       end
     end
 
-    context 'when one move fails' do
-      it 'rollsback repositories to original name' do
-        from_name = project.disk_path
-        to_name = hashed_storage.disk_path
-        allow(service).to receive(:move_repository).and_call_original
-        allow(service).to receive(:move_repository).with(from_name, to_name).once { false } # will disable first move only
+    context 'attachments migration' do
+      let(:attachments_service) { Projects::HashedStorage::MigrateAttachmentsService.new(project, subject.logger) }
 
-        expect(service).to receive(:rollback_folder_move).and_call_original
+      it 'delegates migration to Projects::HashedStorage::MigrateRepositoryService' do
+        expect(Projects::HashedStorage::MigrateAttachmentsService).to receive(:new).with(project, subject.logger).and_return(attachments_service)
+        expect(attachments_service).to receive(:execute)
 
         service.execute
-
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.git")).to be_falsey
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.wiki.git")).to be_falsey
       end
 
-      context 'when rollback fails' do
-        before do
-          from_name = legacy_storage.disk_path
-          to_name = hashed_storage.disk_path
+      it 'does not delegate migration if attachments are already migrated' do
+        project.storage_version = ::Project::LATEST_STORAGE_VERSION
+        expect(Projects::HashedStorage::MigrateAttachmentsService).not_to receive(:new)
 
-          hashed_storage.ensure_storage_path_exists
-          gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
-        end
-
-        it 'does not try to move nil repository over hashed' do
-          expect_move_repository("#{project.disk_path}.wiki", "#{hashed_storage.disk_path}.wiki")
-
-          service.execute
-        end
+        service.execute
       end
     end
-
-    def expect_move_repository(from_name, to_name)
-      expect(gitlab_shell).to receive(:mv_repository).with(project.repository_storage_path, from_name, to_name).and_call_original
-    end
   end
 end
diff --git a/spec/workers/project_migrate_hashed_storage_worker_spec.rb b/spec/workers/project_migrate_hashed_storage_worker_spec.rb
index f5226dee0ad..2e3951e7afc 100644
--- a/spec/workers/project_migrate_hashed_storage_worker_spec.rb
+++ b/spec/workers/project_migrate_hashed_storage_worker_spec.rb
@@ -1,29 +1,53 @@
 require 'spec_helper'
 
-describe ProjectMigrateHashedStorageWorker do
+describe ProjectMigrateHashedStorageWorker, :clean_gitlab_redis_shared_state do
   describe '#perform' do
     let(:project) { create(:project, :empty_repo) }
     let(:pending_delete_project) { create(:project, :empty_repo, pending_delete: true) }
 
-    it 'skips when project no longer exists' do
-      nonexistent_id = 999999999999
+    context 'when have exclusive lease' do
+      before do
+        lease = subject.lease_for(project.id)
 
-      expect(::Projects::HashedStorageMigrationService).not_to receive(:new)
-      subject.perform(nonexistent_id)
-    end
+        allow(Gitlab::ExclusiveLease).to receive(:new).and_return(lease)
+        allow(lease).to receive(:try_obtain).and_return(true)
+      end
+
+      it 'skips when project no longer exists' do
+        nonexistent_id = 999999999999
+
+        expect(::Projects::HashedStorageMigrationService).not_to receive(:new)
+        subject.perform(nonexistent_id)
+      end
+
+      it 'skips when project is pending delete' do
+        expect(::Projects::HashedStorageMigrationService).not_to receive(:new)
 
-    it 'skips when project is pending delete' do
-      expect(::Projects::HashedStorageMigrationService).not_to receive(:new)
+        subject.perform(pending_delete_project.id)
+      end
 
-      subject.perform(pending_delete_project.id)
+      it 'delegates removal to service class' do
+        service = double('service')
+        expect(::Projects::HashedStorageMigrationService).to receive(:new).with(project, subject.logger).and_return(service)
+        expect(service).to receive(:execute)
+
+        subject.perform(project.id)
+      end
     end
 
-    it 'delegates removal to service class' do
-      service = double('service')
-      expect(::Projects::HashedStorageMigrationService).to receive(:new).with(project, subject.logger).and_return(service)
-      expect(service).to receive(:execute)
+    context 'when dont have exclusive lease' do
+      before do
+        lease = subject.lease_for(project.id)
+
+        allow(Gitlab::ExclusiveLease).to receive(:new).and_return(lease)
+        allow(lease).to receive(:try_obtain).and_return(false)
+      end
+
+      it 'skips when dont have lease' do
+        expect(::Projects::HashedStorageMigrationService).not_to receive(:new)
 
-      subject.perform(project.id)
+        subject.perform(project.id)
+      end
     end
   end
 end