diff options
Diffstat (limited to 'lib/gitlab/database/background_migration')
4 files changed, 32 insertions, 12 deletions
diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 185b6d9629f..f3160679d64 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -3,7 +3,9 @@ module Gitlab module Database module BackgroundMigration - class BatchedJob < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + SplitAndRetryError = Class.new(StandardError) + + class BatchedJob < SharedModel include EachBatch include FromUnion @@ -11,6 +13,8 @@ module Gitlab MAX_ATTEMPTS = 3 STUCK_JOBS_TIMEOUT = 1.hour.freeze + TIMEOUT_EXCEPTIONS = [ActiveRecord::StatementTimeout, ActiveRecord::ConnectionTimeoutError, + ActiveRecord::AdapterTimeout, ActiveRecord::LockWaitTimeout].freeze belongs_to :batched_migration, foreign_key: :batched_background_migration_id has_many :batched_job_transition_logs, foreign_key: :batched_background_migration_job_id @@ -51,6 +55,16 @@ module Gitlab job.metrics = {} end + after_transition any => :failed do |job, transition| + error_hash = transition.args.find { |arg| arg[:error].present? } + + exception = error_hash&.fetch(:error) + + job.split_and_retry! if job.can_split?(exception) + rescue SplitAndRetryError => error + Gitlab::AppLogger.error(message: error.message, batched_job_id: job.id) + end + after_transition do |job, transition| error_hash = transition.args.find { |arg| arg[:error].present? } @@ -79,20 +93,25 @@ module Gitlab duration.to_f / batched_migration.interval end + def can_split?(exception) + attempts >= MAX_ATTEMPTS && TIMEOUT_EXCEPTIONS.include?(exception&.class) && batch_size > sub_batch_size + end + def split_and_retry! with_lock do - raise 'Only failed jobs can be split' unless failed? + raise SplitAndRetryError, 'Only failed jobs can be split' unless failed? new_batch_size = batch_size / 2 - raise 'Job cannot be split further' if new_batch_size < 1 + raise SplitAndRetryError, 'Job cannot be split further' if new_batch_size < 1 - batching_strategy = batched_migration.batch_class.new + batching_strategy = batched_migration.batch_class.new(connection: self.class.connection) next_batch_bounds = batching_strategy.next_batch( batched_migration.table_name, batched_migration.column_name, batch_min_value: min_value, - batch_size: new_batch_size + batch_size: new_batch_size, + job_arguments: batched_migration.job_arguments ) midpoint = next_batch_bounds.last diff --git a/lib/gitlab/database/background_migration/batched_job_transition_log.rb b/lib/gitlab/database/background_migration/batched_job_transition_log.rb index 418bf1a101f..55a391005a2 100644 --- a/lib/gitlab/database/background_migration/batched_job_transition_log.rb +++ b/lib/gitlab/database/background_migration/batched_job_transition_log.rb @@ -3,7 +3,7 @@ module Gitlab module Database module BackgroundMigration - class BatchedJobTransitionLog < ApplicationRecord + class BatchedJobTransitionLog < SharedModel include PartitionedTable self.table_name = :batched_background_migration_job_transition_logs diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 1f8ca982ed5..65c15795de6 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -3,7 +3,7 @@ module Gitlab module Database module BackgroundMigration - class BatchedMigration < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + class BatchedMigration < SharedModel JOB_CLASS_MODULE = 'Gitlab::BackgroundMigration' BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies" diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 9308bae20cf..06cd40f1e06 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -6,12 +6,13 @@ module Gitlab class BatchedMigrationRunner FailedToFinalize = Class.new(RuntimeError) - def self.finalize(job_class_name, table_name, column_name, job_arguments) - new.finalize(job_class_name, table_name, column_name, job_arguments) + def self.finalize(job_class_name, table_name, column_name, job_arguments, connection: ApplicationRecord.connection) + new(connection: connection).finalize(job_class_name, table_name, column_name, job_arguments) end - def initialize(migration_wrapper = BatchedMigrationWrapper.new) + def initialize(migration_wrapper = BatchedMigrationWrapper.new, connection: ApplicationRecord.connection) @migration_wrapper = migration_wrapper + @connection = connection end # Runs the next batched_job for a batched_background_migration. @@ -77,7 +78,7 @@ module Gitlab private - attr_reader :migration_wrapper + attr_reader :migration_wrapper, :connection def find_or_create_next_batched_job(active_migration) if next_batch_range = find_next_batch_range(active_migration) @@ -88,7 +89,7 @@ module Gitlab end def find_next_batch_range(active_migration) - batching_strategy = active_migration.batch_class.new + batching_strategy = active_migration.batch_class.new(connection: connection) batch_min_value = active_migration.next_min_value next_batch_bounds = batching_strategy.next_batch( |