diff options
Diffstat (limited to 'app/services/pages/migrate_from_legacy_storage_service.rb')
-rw-r--r-- | app/services/pages/migrate_from_legacy_storage_service.rb | 79 |
1 files changed, 79 insertions, 0 deletions
diff --git a/app/services/pages/migrate_from_legacy_storage_service.rb b/app/services/pages/migrate_from_legacy_storage_service.rb new file mode 100644 index 00000000000..9b36b3f11b4 --- /dev/null +++ b/app/services/pages/migrate_from_legacy_storage_service.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Pages + class MigrateFromLegacyStorageService + def initialize(logger, migration_threads:, batch_size:, ignore_invalid_entries:) + @logger = logger + @migration_threads = migration_threads + @batch_size = batch_size + @ignore_invalid_entries = ignore_invalid_entries + + @migrated = 0 + @errored = 0 + @counters_lock = Mutex.new + end + + def execute + @queue = SizedQueue.new(1) + + threads = start_migration_threads + + ProjectPagesMetadatum.only_on_legacy_storage.each_batch(of: @batch_size) do |batch| + @queue.push(batch) + end + + @queue.close + + @logger.info("Waiting for threads to finish...") + threads.each(&:join) + + { migrated: @migrated, errored: @errored } + end + + def start_migration_threads + Array.new(@migration_threads) do + Thread.new do + while batch = @queue.pop + Rails.application.executor.wrap do + process_batch(batch) + end + end + end + end + end + + def process_batch(batch) + batch.with_project_route_and_deployment.each do |metadatum| + project = metadatum.project + + migrate_project(project) + end + + @logger.info("#{@migrated} projects are migrated successfully, #{@errored} projects failed to be migrated") + rescue => e + # This method should never raise exception otherwise all threads might be killed + # and this will result in queue starving (and deadlock) + Gitlab::ErrorTracking.track_exception(e) + @logger.error("failed processing a batch: #{e.message}") + end + + def migrate_project(project) + result = nil + time = Benchmark.realtime do + result = ::Pages::MigrateLegacyStorageToDeploymentService.new(project, ignore_invalid_entries: @ignore_invalid_entries).execute + end + + if result[:status] == :success + @logger.info("project_id: #{project.id} #{project.pages_path} has been migrated in #{time.round(2)} seconds") + @counters_lock.synchronize { @migrated += 1 } + else + @logger.error("project_id: #{project.id} #{project.pages_path} failed to be migrated in #{time.round(2)} seconds: #{result[:message]}") + @counters_lock.synchronize { @errored += 1 } + end + rescue => e + @counters_lock.synchronize { @errored += 1 } + @logger.error("project_id: #{project&.id} #{project&.pages_path} failed to be migrated: #{e.message}") + Gitlab::ErrorTracking.track_exception(e, project_id: project&.id) + end + end +end |