diff options
Diffstat (limited to 'app/services/pages/migrate_from_legacy_storage_service.rb')
-rw-r--r-- | app/services/pages/migrate_from_legacy_storage_service.rb | 43 |
1 files changed, 27 insertions, 16 deletions
diff --git a/app/services/pages/migrate_from_legacy_storage_service.rb b/app/services/pages/migrate_from_legacy_storage_service.rb index 9b36b3f11b4..b6aa08bba01 100644 --- a/app/services/pages/migrate_from_legacy_storage_service.rb +++ b/app/services/pages/migrate_from_legacy_storage_service.rb @@ -2,36 +2,45 @@ module Pages class MigrateFromLegacyStorageService - def initialize(logger, migration_threads:, batch_size:, ignore_invalid_entries:) + def initialize(logger, ignore_invalid_entries:, mark_projects_as_not_deployed:) @logger = logger - @migration_threads = migration_threads - @batch_size = batch_size @ignore_invalid_entries = ignore_invalid_entries + @mark_projects_as_not_deployed = mark_projects_as_not_deployed @migrated = 0 @errored = 0 @counters_lock = Mutex.new end - def execute + def execute_with_threads(threads:, batch_size:) @queue = SizedQueue.new(1) - threads = start_migration_threads + migration_threads = start_migration_threads(threads) - ProjectPagesMetadatum.only_on_legacy_storage.each_batch(of: @batch_size) do |batch| + ProjectPagesMetadatum.only_on_legacy_storage.each_batch(of: batch_size) do |batch| @queue.push(batch) end @queue.close - @logger.info("Waiting for threads to finish...") - threads.each(&:join) + @logger.info(message: "Pages legacy storage migration: Waiting for threads to finish...") + migration_threads.each(&:join) { migrated: @migrated, errored: @errored } end - def start_migration_threads - Array.new(@migration_threads) do + def execute_for_batch(project_ids) + batch = ProjectPagesMetadatum.only_on_legacy_storage.where(project_id: project_ids) # rubocop: disable CodeReuse/ActiveRecord + + process_batch(batch) + + { migrated: @migrated, errored: @errored } + end + + private + + def start_migration_threads(count) + Array.new(count) do Thread.new do while batch = @queue.pop Rails.application.executor.wrap do @@ -49,30 +58,32 @@ module Pages migrate_project(project) end - @logger.info("#{@migrated} projects are migrated successfully, #{@errored} projects failed to be migrated") + @logger.info(message: "Pages legacy storage migration: batch processed", migrated: @migrated, errored: @errored) rescue => e # This method should never raise exception otherwise all threads might be killed # and this will result in queue starving (and deadlock) Gitlab::ErrorTracking.track_exception(e) - @logger.error("failed processing a batch: #{e.message}") + @logger.error(message: "Pages legacy storage migration: failed processing a batch: #{e.message}") end def migrate_project(project) result = nil time = Benchmark.realtime do - result = ::Pages::MigrateLegacyStorageToDeploymentService.new(project, ignore_invalid_entries: @ignore_invalid_entries).execute + result = ::Pages::MigrateLegacyStorageToDeploymentService.new(project, + ignore_invalid_entries: @ignore_invalid_entries, + mark_projects_as_not_deployed: @mark_projects_as_not_deployed).execute end if result[:status] == :success - @logger.info("project_id: #{project.id} #{project.pages_path} has been migrated in #{time.round(2)} seconds") + @logger.info(message: "Pages legacy storage migration: project migrated: #{result[:message]}", project_id: project.id, pages_path: project.pages_path, duration: time.round(2)) @counters_lock.synchronize { @migrated += 1 } else - @logger.error("project_id: #{project.id} #{project.pages_path} failed to be migrated in #{time.round(2)} seconds: #{result[:message]}") + @logger.error(message: "Pages legacy storage migration: project failed to be migrated: #{result[:message]}", project_id: project.id, pages_path: project.pages_path, duration: time.round(2)) @counters_lock.synchronize { @errored += 1 } end rescue => e @counters_lock.synchronize { @errored += 1 } - @logger.error("project_id: #{project&.id} #{project&.pages_path} failed to be migrated: #{e.message}") + @logger.error(message: "Pages legacy storage migration: project failed to be migrated: #{result[:message]}", project_id: project&.id, pages_path: project&.pages_path) Gitlab::ErrorTracking.track_exception(e, project_id: project&.id) end end |