diff options
Diffstat (limited to 'lib/gitlab/database/background_migration')
3 files changed, 94 insertions, 5 deletions
diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 9a1dc4ee17d..03bd02d7554 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -44,6 +44,51 @@ module Gitlab # TODO: Switch to individual job interval (prereq: https://gitlab.com/gitlab-org/gitlab/-/issues/328801) duration.to_f / batched_migration.interval end + + def split_and_retry! + with_lock do + raise 'Only failed jobs can be split' unless failed? + + new_batch_size = batch_size / 2 + + raise 'Job cannot be split further' if new_batch_size < 1 + + batching_strategy = batched_migration.batch_class.new + next_batch_bounds = batching_strategy.next_batch( + batched_migration.table_name, + batched_migration.column_name, + batch_min_value: min_value, + batch_size: new_batch_size + ) + midpoint = next_batch_bounds.last + + # We don't want the midpoint to go over the existing max_value because + # those IDs would already be in the next batched migration job. + # This could happen when a lot of records in the current batch are deleted. + # + # In this case, we just lower the batch size so that future calls to this + # method could eventually split the job if it continues to fail. + if midpoint >= max_value + update!(batch_size: new_batch_size, attempts: 0) + else + old_max_value = max_value + + update!( + batch_size: new_batch_size, + max_value: midpoint, + attempts: 0, + started_at: nil, + finished_at: nil, + metrics: {} + ) + + new_record = dup + new_record.min_value = midpoint.next + new_record.max_value = old_max_value + new_record.save! + end + end + end end end end diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 36e89023c86..9d66824da51 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -10,7 +10,7 @@ module Gitlab self.table_name = :batched_background_migrations has_many :batched_jobs, foreign_key: :batched_background_migration_id - has_one :last_job, -> { order(id: :desc) }, + has_one :last_job, -> { order(max_value: :desc) }, class_name: 'Gitlab::Database::BackgroundMigration::BatchedJob', foreign_key: :batched_background_migration_id @@ -29,11 +29,16 @@ module Gitlab paused: 0, active: 1, finished: 3, - failed: 4 + failed: 4, + finalizing: 5 } attribute :pause_ms, :integer, default: 100 + def self.find_for_configuration(job_class_name, table_name, column_name, job_arguments) + for_configuration(job_class_name, table_name, column_name, job_arguments).first + end + def self.active_migration active.queue_order.first end diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 67fe6c536e6..14e3919986e 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -4,6 +4,12 @@ module Gitlab module Database module BackgroundMigration class BatchedMigrationRunner + FailedToFinalize = Class.new(RuntimeError) + + def self.finalize(job_class_name, table_name, column_name, job_arguments) + new.finalize(job_class_name, table_name, column_name, job_arguments) + end + def initialize(migration_wrapper = BatchedMigrationWrapper.new) @migration_wrapper = migration_wrapper end @@ -37,10 +43,35 @@ module Gitlab raise 'this method is not intended for use in real environments' end - while migration.active? - run_migration_job(migration) + run_migration_while(migration, :active) + end - migration.reload_last_job + # Finalize migration for given configuration. + # + # If the migration is already finished, do nothing. Otherwise change its status to `finalizing` + # in order to prevent it being picked up by the background worker. Perform all pending jobs, + # then keep running until migration is finished. + def finalize(job_class_name, table_name, column_name, job_arguments) + migration = BatchedMigration.find_for_configuration(job_class_name, table_name, column_name, job_arguments) + + configuration = { + job_class_name: job_class_name, + table_name: table_name, + column_name: column_name, + job_arguments: job_arguments + } + + if migration.nil? + Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" + elsif migration.finished? + Gitlab::AppLogger.warn "Batched background migration for the given configuration is already finished: #{configuration}" + else + migration.finalizing! + migration.batched_jobs.pending.each { |job| migration_wrapper.perform(job) } + + run_migration_while(migration, :finalizing) + + raise FailedToFinalize unless migration.finished? end end @@ -90,6 +121,14 @@ module Gitlab active_migration.finished! end end + + def run_migration_while(migration, status) + while migration.status == status.to_s + run_migration_job(migration) + + migration.reload_last_job + end + end end end end |