diff options
Diffstat (limited to 'app')
-rw-r--r-- | app/models/project.rb | 5 | ||||
-rw-r--r-- | app/models/storage/hashed_project.rb | 1 | ||||
-rw-r--r-- | app/services/projects/hashed_storage/migrate_attachments_service.rb | 54 | ||||
-rw-r--r-- | app/services/projects/hashed_storage/migrate_repository_service.rb | 70 | ||||
-rw-r--r-- | app/services/projects/hashed_storage_migration_service.rb | 62 | ||||
-rw-r--r-- | app/uploaders/file_uploader.rb | 11 | ||||
-rw-r--r-- | app/workers/project_migrate_hashed_storage_worker.rb | 26 |
7 files changed, 169 insertions, 60 deletions
diff --git a/app/models/project.rb b/app/models/project.rb index e276bd2422d..85d580fe0fa 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -273,8 +273,9 @@ class Project < ActiveRecord::Base scope :pending_delete, -> { where(pending_delete: true) } scope :without_deleted, -> { where(pending_delete: false) } - scope :with_hashed_storage, -> { where('storage_version >= 1') } - scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) } + scope :with_storage_feature, ->(feature) { where('storage_version >= :version', version: HASHED_STORAGE_FEATURES[feature]) } + scope :without_storage_feature, ->(feature) { where('storage_version < :version OR storage_version IS NULL', version: HASHED_STORAGE_FEATURES[feature]) } + scope :with_unmigrated_storage, -> { where('storage_version < :version OR storage_version IS NULL', version: LATEST_STORAGE_VERSION) } scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) } scope :sorted_by_stars, -> { reorder('projects.star_count DESC') } diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb index f025f40994e..fae1b64961a 100644 --- a/app/models/storage/hashed_project.rb +++ b/app/models/storage/hashed_project.rb @@ -4,7 +4,6 @@ module Storage delegate :gitlab_shell, :repository_storage_path, to: :project ROOT_PATH_PREFIX = '@hashed'.freeze - STORAGE_VERSION = 1 def initialize(project) @project = project diff --git a/app/services/projects/hashed_storage/migrate_attachments_service.rb b/app/services/projects/hashed_storage/migrate_attachments_service.rb new file mode 100644 index 00000000000..f8aaec8a9c0 --- /dev/null +++ b/app/services/projects/hashed_storage/migrate_attachments_service.rb @@ -0,0 +1,54 @@ +module Projects + module HashedStorage + AttachmentMigrationError = Class.new(StandardError) + + class MigrateAttachmentsService < BaseService + attr_reader :logger, :old_path, :new_path + + def initialize(project, logger = nil) + @project = project + @logger = logger || Rails.logger + end + + def execute + @old_path = project.full_path + @new_path = project.disk_path + + origin = FileUploader.dynamic_path_segment(project) + project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments] + target = FileUploader.dynamic_path_segment(project) + + result = move_folder!(origin, target) + project.save! + + if result && block_given? + yield + end + + result + end + + private + + def move_folder!(old_path, new_path) + unless File.directory?(old_path) + logger.info("Skipped attachments migration from '#{old_path}' to '#{new_path}', source path doesn't exist or is not a directory (PROJECT_ID=#{project.id})") + return + end + + if File.exist?(new_path) + logger.error("Cannot migrate attachments from '#{old_path}' to '#{new_path}', target path already exist (PROJECT_ID=#{project.id})") + raise AttachmentMigrationError, "Target path '#{new_path}' already exist" + end + + # Create hashed storage base path folder + FileUtils.mkdir_p(File.dirname(new_path)) + + FileUtils.mv(old_path, new_path) + logger.info("Migrated project attachments from '#{old_path}' to '#{new_path}' (PROJECT_ID=#{project.id})") + + true + end + end + end +end diff --git a/app/services/projects/hashed_storage/migrate_repository_service.rb b/app/services/projects/hashed_storage/migrate_repository_service.rb new file mode 100644 index 00000000000..7212e7524ab --- /dev/null +++ b/app/services/projects/hashed_storage/migrate_repository_service.rb @@ -0,0 +1,70 @@ +module Projects + module HashedStorage + class MigrateRepositoryService < BaseService + include Gitlab::ShellAdapter + + attr_reader :old_disk_path, :new_disk_path, :old_wiki_disk_path, :old_storage_version, :logger + + def initialize(project, logger = nil) + @project = project + @logger = logger || Rails.logger + end + + def execute + @old_disk_path = project.disk_path + has_wiki = project.wiki.repository_exists? + + @old_storage_version = project.storage_version + project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:repository] + project.ensure_storage_path_exists + + @new_disk_path = project.disk_path + + result = move_repository(@old_disk_path, @new_disk_path) + + if has_wiki + @old_wiki_disk_path = "#{@old_disk_path}.wiki" + result &&= move_repository("#{@old_wiki_disk_path}", "#{@new_disk_path}.wiki") + end + + unless result + rollback_folder_move + project.storage_version = nil + end + + project.repository_read_only = false + project.save! + + if result && block_given? + yield + end + + result + end + + private + + def move_repository(from_name, to_name) + from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git") + to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git") + + # If we don't find the repository on either original or target we should log that as it could be an issue if the + # project was not originally empty. + if !from_exists && !to_exists + logger.warn "Can't find a repository on either source or target paths for #{project.full_path} (ID=#{project.id}) ..." + return false + elsif !from_exists + # Repository have been moved already. + return true + end + + gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name) + end + + def rollback_folder_move + move_repository(@new_disk_path, @old_disk_path) + move_repository("#{@new_disk_path}.wiki", "#{@old_disk_path}.wiki") + end + end + end +end diff --git a/app/services/projects/hashed_storage_migration_service.rb b/app/services/projects/hashed_storage_migration_service.rb index f5945f3b87f..662702c1db5 100644 --- a/app/services/projects/hashed_storage_migration_service.rb +++ b/app/services/projects/hashed_storage_migration_service.rb @@ -1,68 +1,22 @@ module Projects class HashedStorageMigrationService < BaseService - include Gitlab::ShellAdapter - - attr_reader :old_disk_path, :new_disk_path + attr_reader :logger def initialize(project, logger = nil) @project = project - @logger ||= Rails.logger + @logger = logger || Rails.logger end def execute - return if project.hashed_storage?(:repository) - - @old_disk_path = project.disk_path - has_wiki = project.wiki.repository_exists? - - project.storage_version = Storage::HashedProject::STORAGE_VERSION - project.ensure_storage_path_exists - - @new_disk_path = project.disk_path - - result = move_repository(@old_disk_path, @new_disk_path) - - if has_wiki - result &&= move_repository("#{@old_disk_path}.wiki", "#{@new_disk_path}.wiki") - end - - unless result - rollback_folder_move - return + # Migrate repository from Legacy to Hashed Storage + unless project.hashed_storage?(:repository) + return unless HashedStorage::MigrateRepositoryService.new(project, logger).execute end - project.repository_read_only = false - project.save! - - block_given? ? yield : result - end - - private - - def move_repository(from_name, to_name) - from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git") - to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git") - - # If we don't find the repository on either original or target we should log that as it could be an issue if the - # project was not originally empty. - if !from_exists && !to_exists - logger.warn "Can't find a repository on either source or target paths for #{project.full_path} (ID=#{project.id}) ..." - return false - elsif !from_exists - # Repository have been moved already. - return true + # Migrate attachments from Legacy to Hashed Storage + unless project.hashed_storage?(:attachments) + HashedStorage::MigrateAttachmentsService.new(project, logger).execute end - - gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name) - end - - def rollback_folder_move - move_repository(@new_disk_path, @old_disk_path) - move_repository("#{@new_disk_path}.wiki", "#{@old_disk_path}.wiki") - end - - def logger - @logger end end end diff --git a/app/uploaders/file_uploader.rb b/app/uploaders/file_uploader.rb index f4a5cf75018..71658df5b41 100644 --- a/app/uploaders/file_uploader.rb +++ b/app/uploaders/file_uploader.rb @@ -31,12 +31,19 @@ class FileUploader < GitlabUploader # Returns a String without a trailing slash def self.dynamic_path_segment(project) if project.hashed_storage?(:attachments) - File.join(CarrierWave.root, base_dir, project.disk_path) + dynamic_path_builder(project.disk_path) else - File.join(CarrierWave.root, base_dir, project.full_path) + dynamic_path_builder(project.full_path) end end + # Auxiliary method to build dynamic path segment when not using a project model + # + # Prefer to use the `.dynamic_path_segment` as it includes Hashed Storage specific logic + def self.dynamic_path_builder(path) + File.join(CarrierWave.root, base_dir, path) + end + attr_accessor :model attr_reader :secret diff --git a/app/workers/project_migrate_hashed_storage_worker.rb b/app/workers/project_migrate_hashed_storage_worker.rb index ca276d7801c..127aa6b9d7d 100644 --- a/app/workers/project_migrate_hashed_storage_worker.rb +++ b/app/workers/project_migrate_hashed_storage_worker.rb @@ -2,10 +2,34 @@ class ProjectMigrateHashedStorageWorker include Sidekiq::Worker include DedicatedSidekiqQueue + LEASE_TIMEOUT = 30.seconds.to_i + def perform(project_id) project = Project.find_by(id: project_id) return if project.nil? || project.pending_delete? - ::Projects::HashedStorageMigrationService.new(project, logger).execute + uuid = lease_for(project_id).try_obtain + if uuid + ::Projects::HashedStorageMigrationService.new(project, logger).execute + else + false + end + rescue => ex + cancel_lease_for(project_id, uuid) if uuid + raise ex + end + + def lease_for(project_id) + Gitlab::ExclusiveLease.new(lease_key(project_id), timeout: LEASE_TIMEOUT) + end + + private + + def lease_key(project_id) + "project_migrate_hashed_storage_worker:#{project_id}" + end + + def cancel_lease_for(project_id, uuid) + Gitlab::ExclusiveLease.cancel(lease_key(project_id), uuid) end end |