Merge branch 'hashed-storage-attachments-migration-path' into 'master'

Hashed storage attachments migration path

See merge request gitlab-org/gitlab-ce!15352

7 files changed, 169 insertions, 60 deletions

diff --git a/app/models/project.rb b/app/models/project.rb
index e276bd2422d..85d580fe0fa 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -273,8 +273,9 @@ class Project < ActiveRecord::Base
   scope :pending_delete, -> { where(pending_delete: true) }
   scope :without_deleted, -> { where(pending_delete: false) }
 
-  scope :with_hashed_storage, -> { where('storage_version >= 1') }
-  scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) }
+  scope :with_storage_feature, ->(feature) { where('storage_version >= :version', version: HASHED_STORAGE_FEATURES[feature]) }
+  scope :without_storage_feature, ->(feature) { where('storage_version < :version OR storage_version IS NULL', version: HASHED_STORAGE_FEATURES[feature]) }
+  scope :with_unmigrated_storage, -> { where('storage_version < :version OR storage_version IS NULL', version: LATEST_STORAGE_VERSION) }
 
   scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) }
   scope :sorted_by_stars, -> { reorder('projects.star_count DESC') }
diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb
index f025f40994e..fae1b64961a 100644
--- a/app/models/storage/hashed_project.rb
+++ b/app/models/storage/hashed_project.rb
@@ -4,7 +4,6 @@ module Storage
     delegate :gitlab_shell, :repository_storage_path, to: :project
 
     ROOT_PATH_PREFIX = '@hashed'.freeze
-    STORAGE_VERSION = 1
 
     def initialize(project)
       @project = project
diff --git a/app/services/projects/hashed_storage/migrate_attachments_service.rb b/app/services/projects/hashed_storage/migrate_attachments_service.rb
new file mode 100644
index 00000000000..f8aaec8a9c0
--- /dev/null
+++ b/app/services/projects/hashed_storage/migrate_attachments_service.rb
@@ -0,0 +1,54 @@
+module Projects
+  module HashedStorage
+    AttachmentMigrationError = Class.new(StandardError)
+
+    class MigrateAttachmentsService < BaseService
+      attr_reader :logger, :old_path, :new_path
+
+      def initialize(project, logger = nil)
+        @project = project
+        @logger = logger || Rails.logger
+      end
+
+      def execute
+        @old_path = project.full_path
+        @new_path = project.disk_path
+
+        origin = FileUploader.dynamic_path_segment(project)
+        project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments]
+        target = FileUploader.dynamic_path_segment(project)
+
+        result = move_folder!(origin, target)
+        project.save!
+
+        if result && block_given?
+          yield
+        end
+
+        result
+      end
+
+      private
+
+      def move_folder!(old_path, new_path)
+        unless File.directory?(old_path)
+          logger.info("Skipped attachments migration from '#{old_path}' to '#{new_path}', source path doesn't exist or is not a directory (PROJECT_ID=#{project.id})")
+          return
+        end
+
+        if File.exist?(new_path)
+          logger.error("Cannot migrate attachments from '#{old_path}' to '#{new_path}', target path already exist (PROJECT_ID=#{project.id})")
+          raise AttachmentMigrationError, "Target path '#{new_path}' already exist"
+        end
+
+        # Create hashed storage base path folder
+        FileUtils.mkdir_p(File.dirname(new_path))
+
+        FileUtils.mv(old_path, new_path)
+        logger.info("Migrated project attachments from '#{old_path}' to '#{new_path}' (PROJECT_ID=#{project.id})")
+
+        true
+      end
+    end
+  end
+end
diff --git a/app/services/projects/hashed_storage/migrate_repository_service.rb b/app/services/projects/hashed_storage/migrate_repository_service.rb
new file mode 100644
index 00000000000..7212e7524ab
--- /dev/null
+++ b/app/services/projects/hashed_storage/migrate_repository_service.rb
@@ -0,0 +1,70 @@
+module Projects
+  module HashedStorage
+    class MigrateRepositoryService < BaseService
+      include Gitlab::ShellAdapter
+
+      attr_reader :old_disk_path, :new_disk_path, :old_wiki_disk_path, :old_storage_version, :logger
+
+      def initialize(project, logger = nil)
+        @project = project
+        @logger = logger || Rails.logger
+      end
+
+      def execute
+        @old_disk_path = project.disk_path
+        has_wiki = project.wiki.repository_exists?
+
+        @old_storage_version = project.storage_version
+        project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:repository]
+        project.ensure_storage_path_exists
+
+        @new_disk_path = project.disk_path
+
+        result = move_repository(@old_disk_path, @new_disk_path)
+
+        if has_wiki
+          @old_wiki_disk_path = "#{@old_disk_path}.wiki"
+          result &&= move_repository("#{@old_wiki_disk_path}", "#{@new_disk_path}.wiki")
+        end
+
+        unless result
+          rollback_folder_move
+          project.storage_version = nil
+        end
+
+        project.repository_read_only = false
+        project.save!
+
+        if result && block_given?
+          yield
+        end
+
+        result
+      end
+
+      private
+
+      def move_repository(from_name, to_name)
+        from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git")
+        to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git")
+
+        # If we don't find the repository on either original or target we should log that as it could be an issue if the
+        # project was not originally empty.
+        if !from_exists && !to_exists
+          logger.warn "Can't find a repository on either source or target paths for #{project.full_path} (ID=#{project.id}) ..."
+          return false
+        elsif !from_exists
+          # Repository have been moved already.
+          return true
+        end
+
+        gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
+      end
+
+      def rollback_folder_move
+        move_repository(@new_disk_path, @old_disk_path)
+        move_repository("#{@new_disk_path}.wiki", "#{@old_disk_path}.wiki")
+      end
+    end
+  end
+end
diff --git a/app/services/projects/hashed_storage_migration_service.rb b/app/services/projects/hashed_storage_migration_service.rb
index f5945f3b87f..662702c1db5 100644
--- a/app/services/projects/hashed_storage_migration_service.rb
+++ b/app/services/projects/hashed_storage_migration_service.rb
@@ -1,68 +1,22 @@
 module Projects
   class HashedStorageMigrationService < BaseService
-    include Gitlab::ShellAdapter
-
-    attr_reader :old_disk_path, :new_disk_path
+    attr_reader :logger
 
     def initialize(project, logger = nil)
       @project = project
-      @logger ||= Rails.logger
+      @logger = logger || Rails.logger
     end
 
     def execute
-      return if project.hashed_storage?(:repository)
-
-      @old_disk_path = project.disk_path
-      has_wiki = project.wiki.repository_exists?
-
-      project.storage_version = Storage::HashedProject::STORAGE_VERSION
-      project.ensure_storage_path_exists
-
-      @new_disk_path = project.disk_path
-
-      result = move_repository(@old_disk_path, @new_disk_path)
-
-      if has_wiki
-        result &&= move_repository("#{@old_disk_path}.wiki", "#{@new_disk_path}.wiki")
-      end
-
-      unless result
-        rollback_folder_move
-        return
+      # Migrate repository from Legacy to Hashed Storage
+      unless project.hashed_storage?(:repository)
+        return unless HashedStorage::MigrateRepositoryService.new(project, logger).execute
       end
 
-      project.repository_read_only = false
-      project.save!
-
-      block_given? ? yield : result
-    end
-
-    private
-
-    def move_repository(from_name, to_name)
-      from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git")
-      to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git")
-
-      # If we don't find the repository on either original or target we should log that as it could be an issue if the
-      # project was not originally empty.
-      if !from_exists && !to_exists
-        logger.warn "Can't find a repository on either source or target paths for #{project.full_path} (ID=#{project.id}) ..."
-        return false
-      elsif !from_exists
-        # Repository have been moved already.
-        return true
+      # Migrate attachments from Legacy to Hashed Storage
+      unless project.hashed_storage?(:attachments)
+        HashedStorage::MigrateAttachmentsService.new(project, logger).execute
       end
-
-      gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
-    end
-
-    def rollback_folder_move
-      move_repository(@new_disk_path, @old_disk_path)
-      move_repository("#{@new_disk_path}.wiki", "#{@old_disk_path}.wiki")
-    end
-
-    def logger
-      @logger
     end
   end
 end
diff --git a/app/uploaders/file_uploader.rb b/app/uploaders/file_uploader.rb
index f4a5cf75018..71658df5b41 100644
--- a/app/uploaders/file_uploader.rb
+++ b/app/uploaders/file_uploader.rb
@@ -31,12 +31,19 @@ class FileUploader < GitlabUploader
   # Returns a String without a trailing slash
   def self.dynamic_path_segment(project)
     if project.hashed_storage?(:attachments)
-      File.join(CarrierWave.root, base_dir, project.disk_path)
+      dynamic_path_builder(project.disk_path)
     else
-      File.join(CarrierWave.root, base_dir, project.full_path)
+      dynamic_path_builder(project.full_path)
     end
   end
 
+  # Auxiliary method to build dynamic path segment when not using a project model
+  #
+  # Prefer to use the `.dynamic_path_segment` as it includes Hashed Storage specific logic
+  def self.dynamic_path_builder(path)
+    File.join(CarrierWave.root, base_dir, path)
+  end
+
   attr_accessor :model
   attr_reader :secret
 
diff --git a/app/workers/project_migrate_hashed_storage_worker.rb b/app/workers/project_migrate_hashed_storage_worker.rb
index ca276d7801c..127aa6b9d7d 100644
--- a/app/workers/project_migrate_hashed_storage_worker.rb
+++ b/app/workers/project_migrate_hashed_storage_worker.rb
@@ -2,10 +2,34 @@ class ProjectMigrateHashedStorageWorker
   include Sidekiq::Worker
   include DedicatedSidekiqQueue
 
+  LEASE_TIMEOUT = 30.seconds.to_i
+
   def perform(project_id)
     project = Project.find_by(id: project_id)
     return if project.nil? || project.pending_delete?
 
-    ::Projects::HashedStorageMigrationService.new(project, logger).execute
+    uuid = lease_for(project_id).try_obtain
+    if uuid
+      ::Projects::HashedStorageMigrationService.new(project, logger).execute
+    else
+      false
+    end
+  rescue => ex
+    cancel_lease_for(project_id, uuid) if uuid
+    raise ex
+  end
+
+  def lease_for(project_id)
+    Gitlab::ExclusiveLease.new(lease_key(project_id), timeout: LEASE_TIMEOUT)
+  end
+
+  private
+
+  def lease_key(project_id)
+    "project_migrate_hashed_storage_worker:#{project_id}"
+  end
+
+  def cancel_lease_for(project_id, uuid)
+    Gitlab::ExclusiveLease.cancel(lease_key(project_id), uuid)
   end
 end