diff options
Diffstat (limited to 'spec/lib/gitlab/database/background_migration')
6 files changed, 383 insertions, 122 deletions
diff --git a/spec/lib/gitlab/database/background_migration/batch_metrics_spec.rb b/spec/lib/gitlab/database/background_migration/batch_metrics_spec.rb index 66983733411..6db3081ca7e 100644 --- a/spec/lib/gitlab/database/background_migration/batch_metrics_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batch_metrics_spec.rb @@ -10,7 +10,6 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchMetrics do expect(batch_metrics.timings).to be_empty expect(Gitlab::Metrics::System).to receive(:monotonic_time) - .exactly(6).times .and_return(0.0, 111.0, 200.0, 290.0, 300.0, 410.0) batch_metrics.time_operation(:my_label) do @@ -28,4 +27,33 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchMetrics do expect(batch_metrics.timings).to eq(my_label: [111.0, 110.0], my_other_label: [90.0]) end end + + describe '#instrument_operation' do + it 'tracks duration and affected rows' do + expect(batch_metrics.timings).to be_empty + expect(batch_metrics.affected_rows).to be_empty + + expect(Gitlab::Metrics::System).to receive(:monotonic_time) + .and_return(0.0, 111.0, 200.0, 290.0, 300.0, 410.0, 420.0, 450.0) + + batch_metrics.instrument_operation(:my_label) do + 3 + end + + batch_metrics.instrument_operation(:my_other_label) do + 42 + end + + batch_metrics.instrument_operation(:my_label) do + 2 + end + + batch_metrics.instrument_operation(:my_other_label) do + :not_an_integer + end + + expect(batch_metrics.timings).to eq(my_label: [111.0, 110.0], my_other_label: [90.0, 30.0]) + expect(batch_metrics.affected_rows).to eq(my_label: [3, 2], my_other_label: [42]) + end + end end diff --git a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb index 8c663ff9f8a..c39f6a78e93 100644 --- a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb @@ -21,7 +21,19 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d context 'when a job is running' do it 'logs the transition' do - expect(Gitlab::AppLogger).to receive(:info).with( { batched_job_id: job.id, message: 'BatchedJob transition', new_state: :running, previous_state: :failed } ) + expect(Gitlab::AppLogger).to receive(:info).with( + { + batched_job_id: job.id, + batched_migration_id: job.batched_background_migration_id, + exception_class: nil, + exception_message: nil, + job_arguments: job.batched_migration.job_arguments, + job_class_name: job.batched_migration.job_class_name, + message: 'BatchedJob transition', + new_state: :running, + previous_state: :failed + } + ) expect { job.run! }.to change(job, :started_at) end @@ -31,7 +43,19 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d let(:job) { create(:batched_background_migration_job, :running) } it 'logs the transition' do - expect(Gitlab::AppLogger).to receive(:info).with( { batched_job_id: job.id, message: 'BatchedJob transition', new_state: :succeeded, previous_state: :running } ) + expect(Gitlab::AppLogger).to receive(:info).with( + { + batched_job_id: job.id, + batched_migration_id: job.batched_background_migration_id, + exception_class: nil, + exception_message: nil, + job_arguments: job.batched_migration.job_arguments, + job_class_name: job.batched_migration.job_class_name, + message: 'BatchedJob transition', + new_state: :succeeded, + previous_state: :running + } + ) job.succeed! end @@ -89,7 +113,15 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d end it 'logs the error' do - expect(Gitlab::AppLogger).to receive(:error).with( { message: error_message, batched_job_id: job.id } ) + expect(Gitlab::AppLogger).to receive(:error).with( + { + batched_job_id: job.id, + batched_migration_id: job.batched_background_migration_id, + job_arguments: job.batched_migration.job_arguments, + job_class_name: job.batched_migration.job_class_name, + message: error_message + } + ) job.failure!(error: exception) end @@ -100,13 +132,32 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d let(:job) { create(:batched_background_migration_job, :running) } it 'logs the transition' do - expect(Gitlab::AppLogger).to receive(:info).with( { batched_job_id: job.id, message: 'BatchedJob transition', new_state: :failed, previous_state: :running } ) + expect(Gitlab::AppLogger).to receive(:info).with( + { + batched_job_id: job.id, + batched_migration_id: job.batched_background_migration_id, + exception_class: RuntimeError, + exception_message: 'error', + job_arguments: job.batched_migration.job_arguments, + job_class_name: job.batched_migration.job_class_name, + message: 'BatchedJob transition', + new_state: :failed, + previous_state: :running + } + ) - job.failure! + job.failure!(error: RuntimeError.new('error')) end it 'tracks the exception' do - expect(Gitlab::ErrorTracking).to receive(:track_exception).with(RuntimeError, { batched_job_id: job.id } ) + expect(Gitlab::ErrorTracking).to receive(:track_exception).with( + RuntimeError, + { + batched_job_id: job.id, + job_arguments: job.batched_migration.job_arguments, + job_class_name: job.batched_migration.job_class_name + } + ) job.failure!(error: RuntimeError.new) end @@ -130,13 +181,11 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d describe 'scopes' do let_it_be(:fixed_time) { Time.new(2021, 04, 27, 10, 00, 00, 00) } - let_it_be(:pending_job) { create(:batched_background_migration_job, :pending, updated_at: fixed_time) } - let_it_be(:running_job) { create(:batched_background_migration_job, :running, updated_at: fixed_time) } - let_it_be(:stuck_job) { create(:batched_background_migration_job, :pending, updated_at: fixed_time - described_class::STUCK_JOBS_TIMEOUT) } - let_it_be(:failed_job) { create(:batched_background_migration_job, :failed, attempts: 1) } - - let!(:max_attempts_failed_job) { create(:batched_background_migration_job, :failed, attempts: described_class::MAX_ATTEMPTS) } - let!(:succeeded_job) { create(:batched_background_migration_job, :succeeded) } + let_it_be(:pending_job) { create(:batched_background_migration_job, :pending, created_at: fixed_time - 2.days, updated_at: fixed_time) } + let_it_be(:running_job) { create(:batched_background_migration_job, :running, created_at: fixed_time - 2.days, updated_at: fixed_time) } + let_it_be(:stuck_job) { create(:batched_background_migration_job, :pending, created_at: fixed_time, updated_at: fixed_time - described_class::STUCK_JOBS_TIMEOUT) } + let_it_be(:failed_job) { create(:batched_background_migration_job, :failed, created_at: fixed_time, attempts: 1) } + let_it_be(:max_attempts_failed_job) { create(:batched_background_migration_job, :failed, created_at: fixed_time, attempts: described_class::MAX_ATTEMPTS) } before do travel_to fixed_time @@ -165,6 +214,12 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d expect(described_class.retriable).to contain_exactly(failed_job, stuck_job) end end + + describe '.created_since' do + it 'returns jobs since a given time' do + expect(described_class.created_since(fixed_time)).to contain_exactly(stuck_job, failed_job, max_attempts_failed_job) + end + end end describe 'delegated batched_migration attributes' do @@ -194,6 +249,12 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d expect(batched_job.migration_job_arguments).to eq(batched_migration.job_arguments) end end + + describe '#migration_job_class_name' do + it 'returns the migration job_class_name' do + expect(batched_job.migration_job_class_name).to eq(batched_migration.job_class_name) + end + end end describe '#can_split?' do diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_runner_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_runner_spec.rb index 124d204cb62..f147e8204e6 100644 --- a/spec/lib/gitlab/database/background_migration/batched_migration_runner_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_migration_runner_spec.rb @@ -3,8 +3,16 @@ require 'spec_helper' RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do + let(:connection) { Gitlab::Database.database_base_models[:main].connection } let(:migration_wrapper) { double('test wrapper') } - let(:runner) { described_class.new(migration_wrapper) } + + let(:runner) { described_class.new(connection: connection, migration_wrapper: migration_wrapper) } + + around do |example| + Gitlab::Database::SharedModel.using_connection(connection) do + example.run + end + end describe '#run_migration_job' do shared_examples_for 'it has completed the migration' do @@ -86,6 +94,19 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do end end + context 'when the migration should stop' do + let(:migration) { create(:batched_background_migration, :active) } + + let!(:job) { create(:batched_background_migration_job, :failed, batched_migration: migration) } + + it 'changes the status to failure' do + expect(migration).to receive(:should_stop?).and_return(true) + expect(migration_wrapper).to receive(:perform).and_return(job) + + expect { runner.run_migration_job(migration) }.to change { migration.status_name }.from(:active).to(:failed) + end + end + context 'when the migration has previous jobs' do let!(:event1) { create(:event) } let!(:event2) { create(:event) } @@ -282,7 +303,9 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do end describe '#finalize' do - let(:migration_wrapper) { Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper.new } + let(:migration_wrapper) do + Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper.new(connection: connection) + end let(:migration_helpers) { ActiveRecord::Migration.new } let(:table_name) { :_test_batched_migrations_test_table } @@ -293,8 +316,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do let!(:batched_migration) do create( - :batched_background_migration, - status: migration_status, + :batched_background_migration, migration_status, max_value: 8, batch_size: 2, sub_batch_size: 1, @@ -339,7 +361,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do .with('CopyColumnUsingBackgroundMigrationJob', table_name, column_name, job_arguments) .and_return(batched_migration) - expect(batched_migration).to receive(:finalizing!).and_call_original + expect(batched_migration).to receive(:finalize!).and_call_original expect do runner.finalize( @@ -348,7 +370,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do column_name, job_arguments ) - end.to change { batched_migration.reload.status }.from('active').to('finished') + end.to change { batched_migration.reload.status_name }.from(:active).to(:finished) expect(batched_migration.batched_jobs).to all(be_succeeded) @@ -390,7 +412,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do expect(Gitlab::AppLogger).to receive(:warn) .with("Batched background migration for the given configuration is already finished: #{configuration}") - expect(batched_migration).not_to receive(:finalizing!) + expect(batched_migration).not_to receive(:finalize!) runner.finalize( batched_migration.job_class_name, @@ -417,7 +439,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do expect(Gitlab::AppLogger).to receive(:warn) .with("Could not find batched background migration for the given configuration: #{configuration}") - expect(batched_migration).not_to receive(:finalizing!) + expect(batched_migration).not_to receive(:finalize!) runner.finalize( batched_migration.job_class_name, @@ -431,8 +453,6 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do describe '.finalize' do context 'when the connection is passed' do - let(:connection) { double('connection') } - let(:table_name) { :_test_batched_migrations_test_table } let(:column_name) { :some_id } let(:job_arguments) { [:some, :other, :arguments] } diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb index 803123e8e34..7a433be0e2f 100644 --- a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb @@ -27,28 +27,46 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m it { is_expected.to validate_uniqueness_of(:job_arguments).scoped_to(:job_class_name, :table_name, :column_name) } context 'when there are failed jobs' do - let(:batched_migration) { create(:batched_background_migration, status: :active, total_tuple_count: 100) } + let(:batched_migration) { create(:batched_background_migration, :active, total_tuple_count: 100) } let!(:batched_job) { create(:batched_background_migration_job, :failed, batched_migration: batched_migration) } it 'raises an exception' do - expect { batched_migration.finished! }.to raise_error(ActiveRecord::RecordInvalid) + expect { batched_migration.finish! }.to raise_error(StateMachines::InvalidTransition) - expect(batched_migration.reload.status).to eql 'active' + expect(batched_migration.reload.status_name).to be :active end end context 'when the jobs are completed' do - let(:batched_migration) { create(:batched_background_migration, status: :active, total_tuple_count: 100) } + let(:batched_migration) { create(:batched_background_migration, :active, total_tuple_count: 100) } let!(:batched_job) { create(:batched_background_migration_job, :succeeded, batched_migration: batched_migration) } it 'finishes the migration' do - batched_migration.finished! + batched_migration.finish! - expect(batched_migration.status).to eql 'finished' + expect(batched_migration.status_name).to be :finished end end end + describe 'state machine' do + context 'when a migration is executed' do + let!(:batched_migration) { create(:batched_background_migration) } + + it 'updates the started_at' do + expect { batched_migration.execute! }.to change(batched_migration, :started_at).from(nil).to(Time) + end + end + end + + describe '.valid_status' do + valid_status = [:paused, :active, :finished, :failed, :finalizing] + + it 'returns valid status' do + expect(described_class.valid_status).to eq(valid_status) + end + end + describe '.queue_order' do let!(:migration1) { create(:batched_background_migration) } let!(:migration2) { create(:batched_background_migration) } @@ -61,12 +79,23 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m describe '.active_migration' do let!(:migration1) { create(:batched_background_migration, :finished) } - let!(:migration2) { create(:batched_background_migration, :active) } - let!(:migration3) { create(:batched_background_migration, :active) } - it 'returns the first active migration according to queue order' do - expect(described_class.active_migration).to eq(migration2) - create(:batched_background_migration_job, :succeeded, batched_migration: migration1, batch_size: 1000) + context 'without migrations on hold' do + let!(:migration2) { create(:batched_background_migration, :active) } + let!(:migration3) { create(:batched_background_migration, :active) } + + it 'returns the first active migration according to queue order' do + expect(described_class.active_migration).to eq(migration2) + end + end + + context 'with migrations are on hold' do + let!(:migration2) { create(:batched_background_migration, :active, on_hold_until: 10.minutes.from_now) } + let!(:migration3) { create(:batched_background_migration, :active, on_hold_until: 2.minutes.ago) } + + it 'returns the first active migration that is not on hold according to queue order' do + expect(described_class.active_migration).to eq(migration3) + end end end @@ -287,7 +316,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m it 'moves the status of the migration to active' do retry_failed_jobs - expect(batched_migration.status).to eql 'active' + expect(batched_migration.status_name).to be :active end it 'changes the number of attempts to 0' do @@ -301,8 +330,59 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m it 'moves the status of the migration to active' do retry_failed_jobs - expect(batched_migration.status).to eql 'active' + expect(batched_migration.status_name).to be :active + end + end + end + + describe '#should_stop?' do + subject(:should_stop?) { batched_migration.should_stop? } + + let(:batched_migration) { create(:batched_background_migration, started_at: started_at) } + + before do + stub_const('Gitlab::Database::BackgroundMigration::BatchedMigration::MINIMUM_JOBS', 1) + end + + context 'when the started_at is nil' do + let(:started_at) { nil } + + it { expect(should_stop?).to be_falsey } + end + + context 'when the number of jobs is lesser than the MINIMUM_JOBS' do + let(:started_at) { Time.zone.now - 6.days } + + before do + stub_const('Gitlab::Database::BackgroundMigration::BatchedMigration::MINIMUM_JOBS', 10) + stub_const('Gitlab::Database::BackgroundMigration::BatchedMigration::MAXIMUM_FAILED_RATIO', 0.70) + create_list(:batched_background_migration_job, 1, :succeeded, batched_migration: batched_migration) + create_list(:batched_background_migration_job, 3, :failed, batched_migration: batched_migration) + end + + it { expect(should_stop?).to be_falsey } + end + + context 'when the calculated value is greater than the threshold' do + let(:started_at) { Time.zone.now - 6.days } + + before do + stub_const('Gitlab::Database::BackgroundMigration::BatchedMigration::MAXIMUM_FAILED_RATIO', 0.70) + create_list(:batched_background_migration_job, 1, :succeeded, batched_migration: batched_migration) + create_list(:batched_background_migration_job, 3, :failed, batched_migration: batched_migration) + end + + it { expect(should_stop?).to be_truthy } + end + + context 'when the calculated value is lesser than the threshold' do + let(:started_at) { Time.zone.now - 6.days } + + before do + create_list(:batched_background_migration_job, 2, :succeeded, batched_migration: batched_migration) end + + it { expect(should_stop?).to be_falsey } end end @@ -449,6 +529,20 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m end end + describe '#hold!', :freeze_time do + subject { create(:batched_background_migration) } + + let(:time) { 5.minutes.from_now } + + it 'updates on_hold_until property' do + expect { subject.hold!(until_time: time) }.to change { subject.on_hold_until }.from(nil).to(time) + end + + it 'defaults to 10 minutes' do + expect { subject.hold! }.to change { subject.on_hold_until }.from(nil).to(10.minutes.from_now) + end + end + describe '.for_configuration' do let!(:migration) do create( diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb index d6c984c7adb..6a4ac317cad 100644 --- a/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb @@ -3,8 +3,10 @@ require 'spec_helper' RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, '#perform' do - subject { described_class.new.perform(job_record) } + subject { described_class.new(connection: connection, metrics: metrics_tracker).perform(job_record) } + let(:connection) { Gitlab::Database.database_base_models[:main].connection } + let(:metrics_tracker) { instance_double('::Gitlab::Database::BackgroundMigration::PrometheusMetrics', track: nil) } let(:job_class) { Gitlab::BackgroundMigration::CopyColumnUsingBackgroundMigrationJob } let_it_be(:pause_ms) { 250 } @@ -19,6 +21,12 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' let(:job_instance) { double('job instance', batch_metrics: {}) } + around do |example| + Gitlab::Database::SharedModel.using_connection(connection) do + example.run + end + end + before do allow(job_class).to receive(:new).and_return(job_instance) end @@ -78,86 +86,6 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' end end - context 'reporting prometheus metrics' do - let(:labels) { job_record.batched_migration.prometheus_labels } - - before do - allow(job_instance).to receive(:perform) - end - - it 'reports batch_size' do - expect(described_class.metrics[:gauge_batch_size]).to receive(:set).with(labels, job_record.batch_size) - - subject - end - - it 'reports sub_batch_size' do - expect(described_class.metrics[:gauge_sub_batch_size]).to receive(:set).with(labels, job_record.sub_batch_size) - - subject - end - - it 'reports interval' do - expect(described_class.metrics[:gauge_interval]).to receive(:set).with(labels, job_record.batched_migration.interval) - - subject - end - - it 'reports updated tuples (currently based on batch_size)' do - expect(described_class.metrics[:counter_updated_tuples]).to receive(:increment).with(labels, job_record.batch_size) - - subject - end - - it 'reports migrated tuples' do - count = double - expect(job_record.batched_migration).to receive(:migrated_tuple_count).and_return(count) - expect(described_class.metrics[:gauge_migrated_tuples]).to receive(:set).with(labels, count) - - subject - end - - it 'reports summary of query timings' do - metrics = { 'timings' => { 'update_all' => [1, 2, 3, 4, 5] } } - - expect(job_instance).to receive(:batch_metrics).and_return(metrics) - - metrics['timings'].each do |key, timings| - summary_labels = labels.merge(operation: key) - timings.each do |timing| - expect(described_class.metrics[:histogram_timings]).to receive(:observe).with(summary_labels, timing) - end - end - - subject - end - - it 'reports job duration' do - freeze_time do - expect(Time).to receive(:current).and_return(Time.zone.now - 5.seconds).ordered - allow(Time).to receive(:current).and_call_original - - expect(described_class.metrics[:gauge_job_duration]).to receive(:set).with(labels, 5.seconds) - - subject - end - end - - it 'reports the total tuple count for the migration' do - expect(described_class.metrics[:gauge_total_tuple_count]).to receive(:set).with(labels, job_record.batched_migration.total_tuple_count) - - subject - end - - it 'reports last updated at timestamp' do - freeze_time do - expect(described_class.metrics[:gauge_last_update_time]).to receive(:set).with(labels, Time.current.to_i) - - subject - end - end - end - context 'when the migration job does not raise an error' do it 'marks the tracking record as succeeded' do expect(job_instance).to receive(:perform).with(1, 10, 'events', 'id', 1, pause_ms, 'id', 'other_id') @@ -171,6 +99,13 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' expect(reloaded_job_record.finished_at).to eq(Time.current) end end + + it 'tracks metrics of the execution' do + expect(job_instance).to receive(:perform) + expect(metrics_tracker).to receive(:track).with(job_record) + + subject + end end context 'when the migration job raises an error' do @@ -189,6 +124,13 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' expect(reloaded_job_record.finished_at).to eq(Time.current) end end + + it 'tracks metrics of the execution' do + expect(job_instance).to receive(:perform).and_raise(error_class) + expect(metrics_tracker).to receive(:track).with(job_record) + + expect { subject }.to raise_error(error_class) + end end it_behaves_like 'an error is raised', RuntimeError.new('Something broke!') @@ -203,7 +145,6 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' stub_const('Gitlab::BackgroundMigration::Foo', migration_class) end - let(:connection) { double(:connection) } let(:active_migration) { create(:batched_background_migration, :active, job_class_name: 'Foo') } let!(:job_record) { create(:batched_background_migration_job, batched_migration: active_migration) } @@ -212,12 +153,11 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' expect(job_instance).to receive(:perform) - described_class.new(connection: connection).perform(job_record) + subject end end context 'when the batched background migration inherits from BaseJob' do - let(:connection) { double(:connection) } let(:active_migration) { create(:batched_background_migration, :active, job_class_name: 'Foo') } let!(:job_record) { create(:batched_background_migration_job, batched_migration: active_migration) } @@ -232,7 +172,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' expect(job_instance).to receive(:perform) - described_class.new(connection: connection).perform(job_record) + subject end end end diff --git a/spec/lib/gitlab/database/background_migration/prometheus_metrics_spec.rb b/spec/lib/gitlab/database/background_migration/prometheus_metrics_spec.rb new file mode 100644 index 00000000000..1f256de35ec --- /dev/null +++ b/spec/lib/gitlab/database/background_migration/prometheus_metrics_spec.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::BackgroundMigration::PrometheusMetrics, :prometheus do + describe '#track' do + let(:job_record) do + build(:batched_background_migration_job, :succeeded, + started_at: Time.current - 2.minutes, + finished_at: Time.current - 1.minute, + updated_at: Time.current, + metrics: { 'timings' => { 'update_all' => [0.05, 0.2, 0.4, 0.9, 4] } }) + end + + let(:labels) { job_record.batched_migration.prometheus_labels } + + subject(:track_job_record_metrics) { described_class.new.track(job_record) } + + it 'reports batch_size' do + track_job_record_metrics + + expect(metric_for_job_by_name(:gauge_batch_size)).to eq(job_record.batch_size) + end + + it 'reports sub_batch_size' do + track_job_record_metrics + + expect(metric_for_job_by_name(:gauge_sub_batch_size)).to eq(job_record.sub_batch_size) + end + + it 'reports interval' do + track_job_record_metrics + + expect(metric_for_job_by_name(:gauge_interval)).to eq(job_record.batched_migration.interval) + end + + it 'reports job duration' do + freeze_time do + track_job_record_metrics + + expect(metric_for_job_by_name(:gauge_job_duration)).to eq(1.minute) + end + end + + it 'increments updated tuples (currently based on batch_size)' do + expect(described_class.metrics[:counter_updated_tuples]).to receive(:increment) + .with(labels, job_record.batch_size) + .twice + .and_call_original + + track_job_record_metrics + + expect(metric_for_job_by_name(:counter_updated_tuples)).to eq(job_record.batch_size) + + described_class.new.track(job_record) + + expect(metric_for_job_by_name(:counter_updated_tuples)).to eq(job_record.batch_size * 2) + end + + it 'reports migrated tuples' do + expect(job_record.batched_migration).to receive(:migrated_tuple_count).and_return(20) + + track_job_record_metrics + + expect(metric_for_job_by_name(:gauge_migrated_tuples)).to eq(20) + end + + it 'reports the total tuple count for the migration' do + track_job_record_metrics + + expect(metric_for_job_by_name(:gauge_total_tuple_count)).to eq(job_record.batched_migration.total_tuple_count) + end + + it 'reports last updated at timestamp' do + freeze_time do + track_job_record_metrics + + expect(metric_for_job_by_name(:gauge_last_update_time)).to eq(Time.current.to_i) + end + end + + it 'reports summary of query timings' do + summary_labels = labels.merge(operation: 'update_all') + + job_record.metrics['timings']['update_all'].each do |timing| + expect(described_class.metrics[:histogram_timings]).to receive(:observe) + .with(summary_labels, timing) + .and_call_original + end + + track_job_record_metrics + + expect(metric_for_job_by_name(:histogram_timings, job_labels: summary_labels)) + .to eq({ 0.1 => 1.0, 0.25 => 2.0, 0.5 => 3.0, 1 => 4.0, 5 => 5.0 }) + end + + context 'when the tracking record does not having timing metrics' do + before do + job_record.metrics = {} + end + + it 'does not attempt to report query timings' do + summary_labels = labels.merge(operation: 'update_all') + + expect(described_class.metrics[:histogram_timings]).not_to receive(:observe) + + track_job_record_metrics + + expect(metric_for_job_by_name(:histogram_timings, job_labels: summary_labels)) + .to eq({ 0.1 => 0.0, 0.25 => 0.0, 0.5 => 0.0, 1 => 0.0, 5 => 0.0 }) + end + end + + def metric_for_job_by_name(name, job_labels: labels) + described_class.metrics[name].values[job_labels].get + end + end +end |