diff options
Diffstat (limited to 'spec/lib/gitlab/database')
21 files changed, 838 insertions, 253 deletions
diff --git a/spec/lib/gitlab/database/background_migration/batch_metrics_spec.rb b/spec/lib/gitlab/database/background_migration/batch_metrics_spec.rb index 66983733411..6db3081ca7e 100644 --- a/spec/lib/gitlab/database/background_migration/batch_metrics_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batch_metrics_spec.rb @@ -10,7 +10,6 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchMetrics do expect(batch_metrics.timings).to be_empty expect(Gitlab::Metrics::System).to receive(:monotonic_time) - .exactly(6).times .and_return(0.0, 111.0, 200.0, 290.0, 300.0, 410.0) batch_metrics.time_operation(:my_label) do @@ -28,4 +27,33 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchMetrics do expect(batch_metrics.timings).to eq(my_label: [111.0, 110.0], my_other_label: [90.0]) end end + + describe '#instrument_operation' do + it 'tracks duration and affected rows' do + expect(batch_metrics.timings).to be_empty + expect(batch_metrics.affected_rows).to be_empty + + expect(Gitlab::Metrics::System).to receive(:monotonic_time) + .and_return(0.0, 111.0, 200.0, 290.0, 300.0, 410.0, 420.0, 450.0) + + batch_metrics.instrument_operation(:my_label) do + 3 + end + + batch_metrics.instrument_operation(:my_other_label) do + 42 + end + + batch_metrics.instrument_operation(:my_label) do + 2 + end + + batch_metrics.instrument_operation(:my_other_label) do + :not_an_integer + end + + expect(batch_metrics.timings).to eq(my_label: [111.0, 110.0], my_other_label: [90.0, 30.0]) + expect(batch_metrics.affected_rows).to eq(my_label: [3, 2], my_other_label: [42]) + end + end end diff --git a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb index 8c663ff9f8a..c39f6a78e93 100644 --- a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb @@ -21,7 +21,19 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d context 'when a job is running' do it 'logs the transition' do - expect(Gitlab::AppLogger).to receive(:info).with( { batched_job_id: job.id, message: 'BatchedJob transition', new_state: :running, previous_state: :failed } ) + expect(Gitlab::AppLogger).to receive(:info).with( + { + batched_job_id: job.id, + batched_migration_id: job.batched_background_migration_id, + exception_class: nil, + exception_message: nil, + job_arguments: job.batched_migration.job_arguments, + job_class_name: job.batched_migration.job_class_name, + message: 'BatchedJob transition', + new_state: :running, + previous_state: :failed + } + ) expect { job.run! }.to change(job, :started_at) end @@ -31,7 +43,19 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d let(:job) { create(:batched_background_migration_job, :running) } it 'logs the transition' do - expect(Gitlab::AppLogger).to receive(:info).with( { batched_job_id: job.id, message: 'BatchedJob transition', new_state: :succeeded, previous_state: :running } ) + expect(Gitlab::AppLogger).to receive(:info).with( + { + batched_job_id: job.id, + batched_migration_id: job.batched_background_migration_id, + exception_class: nil, + exception_message: nil, + job_arguments: job.batched_migration.job_arguments, + job_class_name: job.batched_migration.job_class_name, + message: 'BatchedJob transition', + new_state: :succeeded, + previous_state: :running + } + ) job.succeed! end @@ -89,7 +113,15 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d end it 'logs the error' do - expect(Gitlab::AppLogger).to receive(:error).with( { message: error_message, batched_job_id: job.id } ) + expect(Gitlab::AppLogger).to receive(:error).with( + { + batched_job_id: job.id, + batched_migration_id: job.batched_background_migration_id, + job_arguments: job.batched_migration.job_arguments, + job_class_name: job.batched_migration.job_class_name, + message: error_message + } + ) job.failure!(error: exception) end @@ -100,13 +132,32 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d let(:job) { create(:batched_background_migration_job, :running) } it 'logs the transition' do - expect(Gitlab::AppLogger).to receive(:info).with( { batched_job_id: job.id, message: 'BatchedJob transition', new_state: :failed, previous_state: :running } ) + expect(Gitlab::AppLogger).to receive(:info).with( + { + batched_job_id: job.id, + batched_migration_id: job.batched_background_migration_id, + exception_class: RuntimeError, + exception_message: 'error', + job_arguments: job.batched_migration.job_arguments, + job_class_name: job.batched_migration.job_class_name, + message: 'BatchedJob transition', + new_state: :failed, + previous_state: :running + } + ) - job.failure! + job.failure!(error: RuntimeError.new('error')) end it 'tracks the exception' do - expect(Gitlab::ErrorTracking).to receive(:track_exception).with(RuntimeError, { batched_job_id: job.id } ) + expect(Gitlab::ErrorTracking).to receive(:track_exception).with( + RuntimeError, + { + batched_job_id: job.id, + job_arguments: job.batched_migration.job_arguments, + job_class_name: job.batched_migration.job_class_name + } + ) job.failure!(error: RuntimeError.new) end @@ -130,13 +181,11 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d describe 'scopes' do let_it_be(:fixed_time) { Time.new(2021, 04, 27, 10, 00, 00, 00) } - let_it_be(:pending_job) { create(:batched_background_migration_job, :pending, updated_at: fixed_time) } - let_it_be(:running_job) { create(:batched_background_migration_job, :running, updated_at: fixed_time) } - let_it_be(:stuck_job) { create(:batched_background_migration_job, :pending, updated_at: fixed_time - described_class::STUCK_JOBS_TIMEOUT) } - let_it_be(:failed_job) { create(:batched_background_migration_job, :failed, attempts: 1) } - - let!(:max_attempts_failed_job) { create(:batched_background_migration_job, :failed, attempts: described_class::MAX_ATTEMPTS) } - let!(:succeeded_job) { create(:batched_background_migration_job, :succeeded) } + let_it_be(:pending_job) { create(:batched_background_migration_job, :pending, created_at: fixed_time - 2.days, updated_at: fixed_time) } + let_it_be(:running_job) { create(:batched_background_migration_job, :running, created_at: fixed_time - 2.days, updated_at: fixed_time) } + let_it_be(:stuck_job) { create(:batched_background_migration_job, :pending, created_at: fixed_time, updated_at: fixed_time - described_class::STUCK_JOBS_TIMEOUT) } + let_it_be(:failed_job) { create(:batched_background_migration_job, :failed, created_at: fixed_time, attempts: 1) } + let_it_be(:max_attempts_failed_job) { create(:batched_background_migration_job, :failed, created_at: fixed_time, attempts: described_class::MAX_ATTEMPTS) } before do travel_to fixed_time @@ -165,6 +214,12 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d expect(described_class.retriable).to contain_exactly(failed_job, stuck_job) end end + + describe '.created_since' do + it 'returns jobs since a given time' do + expect(described_class.created_since(fixed_time)).to contain_exactly(stuck_job, failed_job, max_attempts_failed_job) + end + end end describe 'delegated batched_migration attributes' do @@ -194,6 +249,12 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d expect(batched_job.migration_job_arguments).to eq(batched_migration.job_arguments) end end + + describe '#migration_job_class_name' do + it 'returns the migration job_class_name' do + expect(batched_job.migration_job_class_name).to eq(batched_migration.job_class_name) + end + end end describe '#can_split?' do diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_runner_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_runner_spec.rb index 124d204cb62..f147e8204e6 100644 --- a/spec/lib/gitlab/database/background_migration/batched_migration_runner_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_migration_runner_spec.rb @@ -3,8 +3,16 @@ require 'spec_helper' RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do + let(:connection) { Gitlab::Database.database_base_models[:main].connection } let(:migration_wrapper) { double('test wrapper') } - let(:runner) { described_class.new(migration_wrapper) } + + let(:runner) { described_class.new(connection: connection, migration_wrapper: migration_wrapper) } + + around do |example| + Gitlab::Database::SharedModel.using_connection(connection) do + example.run + end + end describe '#run_migration_job' do shared_examples_for 'it has completed the migration' do @@ -86,6 +94,19 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do end end + context 'when the migration should stop' do + let(:migration) { create(:batched_background_migration, :active) } + + let!(:job) { create(:batched_background_migration_job, :failed, batched_migration: migration) } + + it 'changes the status to failure' do + expect(migration).to receive(:should_stop?).and_return(true) + expect(migration_wrapper).to receive(:perform).and_return(job) + + expect { runner.run_migration_job(migration) }.to change { migration.status_name }.from(:active).to(:failed) + end + end + context 'when the migration has previous jobs' do let!(:event1) { create(:event) } let!(:event2) { create(:event) } @@ -282,7 +303,9 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do end describe '#finalize' do - let(:migration_wrapper) { Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper.new } + let(:migration_wrapper) do + Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper.new(connection: connection) + end let(:migration_helpers) { ActiveRecord::Migration.new } let(:table_name) { :_test_batched_migrations_test_table } @@ -293,8 +316,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do let!(:batched_migration) do create( - :batched_background_migration, - status: migration_status, + :batched_background_migration, migration_status, max_value: 8, batch_size: 2, sub_batch_size: 1, @@ -339,7 +361,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do .with('CopyColumnUsingBackgroundMigrationJob', table_name, column_name, job_arguments) .and_return(batched_migration) - expect(batched_migration).to receive(:finalizing!).and_call_original + expect(batched_migration).to receive(:finalize!).and_call_original expect do runner.finalize( @@ -348,7 +370,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do column_name, job_arguments ) - end.to change { batched_migration.reload.status }.from('active').to('finished') + end.to change { batched_migration.reload.status_name }.from(:active).to(:finished) expect(batched_migration.batched_jobs).to all(be_succeeded) @@ -390,7 +412,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do expect(Gitlab::AppLogger).to receive(:warn) .with("Batched background migration for the given configuration is already finished: #{configuration}") - expect(batched_migration).not_to receive(:finalizing!) + expect(batched_migration).not_to receive(:finalize!) runner.finalize( batched_migration.job_class_name, @@ -417,7 +439,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do expect(Gitlab::AppLogger).to receive(:warn) .with("Could not find batched background migration for the given configuration: #{configuration}") - expect(batched_migration).not_to receive(:finalizing!) + expect(batched_migration).not_to receive(:finalize!) runner.finalize( batched_migration.job_class_name, @@ -431,8 +453,6 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationRunner do describe '.finalize' do context 'when the connection is passed' do - let(:connection) { double('connection') } - let(:table_name) { :_test_batched_migrations_test_table } let(:column_name) { :some_id } let(:job_arguments) { [:some, :other, :arguments] } diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb index 803123e8e34..7a433be0e2f 100644 --- a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb @@ -27,28 +27,46 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m it { is_expected.to validate_uniqueness_of(:job_arguments).scoped_to(:job_class_name, :table_name, :column_name) } context 'when there are failed jobs' do - let(:batched_migration) { create(:batched_background_migration, status: :active, total_tuple_count: 100) } + let(:batched_migration) { create(:batched_background_migration, :active, total_tuple_count: 100) } let!(:batched_job) { create(:batched_background_migration_job, :failed, batched_migration: batched_migration) } it 'raises an exception' do - expect { batched_migration.finished! }.to raise_error(ActiveRecord::RecordInvalid) + expect { batched_migration.finish! }.to raise_error(StateMachines::InvalidTransition) - expect(batched_migration.reload.status).to eql 'active' + expect(batched_migration.reload.status_name).to be :active end end context 'when the jobs are completed' do - let(:batched_migration) { create(:batched_background_migration, status: :active, total_tuple_count: 100) } + let(:batched_migration) { create(:batched_background_migration, :active, total_tuple_count: 100) } let!(:batched_job) { create(:batched_background_migration_job, :succeeded, batched_migration: batched_migration) } it 'finishes the migration' do - batched_migration.finished! + batched_migration.finish! - expect(batched_migration.status).to eql 'finished' + expect(batched_migration.status_name).to be :finished end end end + describe 'state machine' do + context 'when a migration is executed' do + let!(:batched_migration) { create(:batched_background_migration) } + + it 'updates the started_at' do + expect { batched_migration.execute! }.to change(batched_migration, :started_at).from(nil).to(Time) + end + end + end + + describe '.valid_status' do + valid_status = [:paused, :active, :finished, :failed, :finalizing] + + it 'returns valid status' do + expect(described_class.valid_status).to eq(valid_status) + end + end + describe '.queue_order' do let!(:migration1) { create(:batched_background_migration) } let!(:migration2) { create(:batched_background_migration) } @@ -61,12 +79,23 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m describe '.active_migration' do let!(:migration1) { create(:batched_background_migration, :finished) } - let!(:migration2) { create(:batched_background_migration, :active) } - let!(:migration3) { create(:batched_background_migration, :active) } - it 'returns the first active migration according to queue order' do - expect(described_class.active_migration).to eq(migration2) - create(:batched_background_migration_job, :succeeded, batched_migration: migration1, batch_size: 1000) + context 'without migrations on hold' do + let!(:migration2) { create(:batched_background_migration, :active) } + let!(:migration3) { create(:batched_background_migration, :active) } + + it 'returns the first active migration according to queue order' do + expect(described_class.active_migration).to eq(migration2) + end + end + + context 'with migrations are on hold' do + let!(:migration2) { create(:batched_background_migration, :active, on_hold_until: 10.minutes.from_now) } + let!(:migration3) { create(:batched_background_migration, :active, on_hold_until: 2.minutes.ago) } + + it 'returns the first active migration that is not on hold according to queue order' do + expect(described_class.active_migration).to eq(migration3) + end end end @@ -287,7 +316,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m it 'moves the status of the migration to active' do retry_failed_jobs - expect(batched_migration.status).to eql 'active' + expect(batched_migration.status_name).to be :active end it 'changes the number of attempts to 0' do @@ -301,8 +330,59 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m it 'moves the status of the migration to active' do retry_failed_jobs - expect(batched_migration.status).to eql 'active' + expect(batched_migration.status_name).to be :active + end + end + end + + describe '#should_stop?' do + subject(:should_stop?) { batched_migration.should_stop? } + + let(:batched_migration) { create(:batched_background_migration, started_at: started_at) } + + before do + stub_const('Gitlab::Database::BackgroundMigration::BatchedMigration::MINIMUM_JOBS', 1) + end + + context 'when the started_at is nil' do + let(:started_at) { nil } + + it { expect(should_stop?).to be_falsey } + end + + context 'when the number of jobs is lesser than the MINIMUM_JOBS' do + let(:started_at) { Time.zone.now - 6.days } + + before do + stub_const('Gitlab::Database::BackgroundMigration::BatchedMigration::MINIMUM_JOBS', 10) + stub_const('Gitlab::Database::BackgroundMigration::BatchedMigration::MAXIMUM_FAILED_RATIO', 0.70) + create_list(:batched_background_migration_job, 1, :succeeded, batched_migration: batched_migration) + create_list(:batched_background_migration_job, 3, :failed, batched_migration: batched_migration) + end + + it { expect(should_stop?).to be_falsey } + end + + context 'when the calculated value is greater than the threshold' do + let(:started_at) { Time.zone.now - 6.days } + + before do + stub_const('Gitlab::Database::BackgroundMigration::BatchedMigration::MAXIMUM_FAILED_RATIO', 0.70) + create_list(:batched_background_migration_job, 1, :succeeded, batched_migration: batched_migration) + create_list(:batched_background_migration_job, 3, :failed, batched_migration: batched_migration) + end + + it { expect(should_stop?).to be_truthy } + end + + context 'when the calculated value is lesser than the threshold' do + let(:started_at) { Time.zone.now - 6.days } + + before do + create_list(:batched_background_migration_job, 2, :succeeded, batched_migration: batched_migration) end + + it { expect(should_stop?).to be_falsey } end end @@ -449,6 +529,20 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m end end + describe '#hold!', :freeze_time do + subject { create(:batched_background_migration) } + + let(:time) { 5.minutes.from_now } + + it 'updates on_hold_until property' do + expect { subject.hold!(until_time: time) }.to change { subject.on_hold_until }.from(nil).to(time) + end + + it 'defaults to 10 minutes' do + expect { subject.hold! }.to change { subject.on_hold_until }.from(nil).to(10.minutes.from_now) + end + end + describe '.for_configuration' do let!(:migration) do create( diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb index d6c984c7adb..6a4ac317cad 100644 --- a/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb @@ -3,8 +3,10 @@ require 'spec_helper' RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, '#perform' do - subject { described_class.new.perform(job_record) } + subject { described_class.new(connection: connection, metrics: metrics_tracker).perform(job_record) } + let(:connection) { Gitlab::Database.database_base_models[:main].connection } + let(:metrics_tracker) { instance_double('::Gitlab::Database::BackgroundMigration::PrometheusMetrics', track: nil) } let(:job_class) { Gitlab::BackgroundMigration::CopyColumnUsingBackgroundMigrationJob } let_it_be(:pause_ms) { 250 } @@ -19,6 +21,12 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' let(:job_instance) { double('job instance', batch_metrics: {}) } + around do |example| + Gitlab::Database::SharedModel.using_connection(connection) do + example.run + end + end + before do allow(job_class).to receive(:new).and_return(job_instance) end @@ -78,86 +86,6 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' end end - context 'reporting prometheus metrics' do - let(:labels) { job_record.batched_migration.prometheus_labels } - - before do - allow(job_instance).to receive(:perform) - end - - it 'reports batch_size' do - expect(described_class.metrics[:gauge_batch_size]).to receive(:set).with(labels, job_record.batch_size) - - subject - end - - it 'reports sub_batch_size' do - expect(described_class.metrics[:gauge_sub_batch_size]).to receive(:set).with(labels, job_record.sub_batch_size) - - subject - end - - it 'reports interval' do - expect(described_class.metrics[:gauge_interval]).to receive(:set).with(labels, job_record.batched_migration.interval) - - subject - end - - it 'reports updated tuples (currently based on batch_size)' do - expect(described_class.metrics[:counter_updated_tuples]).to receive(:increment).with(labels, job_record.batch_size) - - subject - end - - it 'reports migrated tuples' do - count = double - expect(job_record.batched_migration).to receive(:migrated_tuple_count).and_return(count) - expect(described_class.metrics[:gauge_migrated_tuples]).to receive(:set).with(labels, count) - - subject - end - - it 'reports summary of query timings' do - metrics = { 'timings' => { 'update_all' => [1, 2, 3, 4, 5] } } - - expect(job_instance).to receive(:batch_metrics).and_return(metrics) - - metrics['timings'].each do |key, timings| - summary_labels = labels.merge(operation: key) - timings.each do |timing| - expect(described_class.metrics[:histogram_timings]).to receive(:observe).with(summary_labels, timing) - end - end - - subject - end - - it 'reports job duration' do - freeze_time do - expect(Time).to receive(:current).and_return(Time.zone.now - 5.seconds).ordered - allow(Time).to receive(:current).and_call_original - - expect(described_class.metrics[:gauge_job_duration]).to receive(:set).with(labels, 5.seconds) - - subject - end - end - - it 'reports the total tuple count for the migration' do - expect(described_class.metrics[:gauge_total_tuple_count]).to receive(:set).with(labels, job_record.batched_migration.total_tuple_count) - - subject - end - - it 'reports last updated at timestamp' do - freeze_time do - expect(described_class.metrics[:gauge_last_update_time]).to receive(:set).with(labels, Time.current.to_i) - - subject - end - end - end - context 'when the migration job does not raise an error' do it 'marks the tracking record as succeeded' do expect(job_instance).to receive(:perform).with(1, 10, 'events', 'id', 1, pause_ms, 'id', 'other_id') @@ -171,6 +99,13 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' expect(reloaded_job_record.finished_at).to eq(Time.current) end end + + it 'tracks metrics of the execution' do + expect(job_instance).to receive(:perform) + expect(metrics_tracker).to receive(:track).with(job_record) + + subject + end end context 'when the migration job raises an error' do @@ -189,6 +124,13 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' expect(reloaded_job_record.finished_at).to eq(Time.current) end end + + it 'tracks metrics of the execution' do + expect(job_instance).to receive(:perform).and_raise(error_class) + expect(metrics_tracker).to receive(:track).with(job_record) + + expect { subject }.to raise_error(error_class) + end end it_behaves_like 'an error is raised', RuntimeError.new('Something broke!') @@ -203,7 +145,6 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' stub_const('Gitlab::BackgroundMigration::Foo', migration_class) end - let(:connection) { double(:connection) } let(:active_migration) { create(:batched_background_migration, :active, job_class_name: 'Foo') } let!(:job_record) { create(:batched_background_migration_job, batched_migration: active_migration) } @@ -212,12 +153,11 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' expect(job_instance).to receive(:perform) - described_class.new(connection: connection).perform(job_record) + subject end end context 'when the batched background migration inherits from BaseJob' do - let(:connection) { double(:connection) } let(:active_migration) { create(:batched_background_migration, :active, job_class_name: 'Foo') } let!(:job_record) { create(:batched_background_migration_job, batched_migration: active_migration) } @@ -232,7 +172,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' expect(job_instance).to receive(:perform) - described_class.new(connection: connection).perform(job_record) + subject end end end diff --git a/spec/lib/gitlab/database/background_migration/prometheus_metrics_spec.rb b/spec/lib/gitlab/database/background_migration/prometheus_metrics_spec.rb new file mode 100644 index 00000000000..1f256de35ec --- /dev/null +++ b/spec/lib/gitlab/database/background_migration/prometheus_metrics_spec.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::BackgroundMigration::PrometheusMetrics, :prometheus do + describe '#track' do + let(:job_record) do + build(:batched_background_migration_job, :succeeded, + started_at: Time.current - 2.minutes, + finished_at: Time.current - 1.minute, + updated_at: Time.current, + metrics: { 'timings' => { 'update_all' => [0.05, 0.2, 0.4, 0.9, 4] } }) + end + + let(:labels) { job_record.batched_migration.prometheus_labels } + + subject(:track_job_record_metrics) { described_class.new.track(job_record) } + + it 'reports batch_size' do + track_job_record_metrics + + expect(metric_for_job_by_name(:gauge_batch_size)).to eq(job_record.batch_size) + end + + it 'reports sub_batch_size' do + track_job_record_metrics + + expect(metric_for_job_by_name(:gauge_sub_batch_size)).to eq(job_record.sub_batch_size) + end + + it 'reports interval' do + track_job_record_metrics + + expect(metric_for_job_by_name(:gauge_interval)).to eq(job_record.batched_migration.interval) + end + + it 'reports job duration' do + freeze_time do + track_job_record_metrics + + expect(metric_for_job_by_name(:gauge_job_duration)).to eq(1.minute) + end + end + + it 'increments updated tuples (currently based on batch_size)' do + expect(described_class.metrics[:counter_updated_tuples]).to receive(:increment) + .with(labels, job_record.batch_size) + .twice + .and_call_original + + track_job_record_metrics + + expect(metric_for_job_by_name(:counter_updated_tuples)).to eq(job_record.batch_size) + + described_class.new.track(job_record) + + expect(metric_for_job_by_name(:counter_updated_tuples)).to eq(job_record.batch_size * 2) + end + + it 'reports migrated tuples' do + expect(job_record.batched_migration).to receive(:migrated_tuple_count).and_return(20) + + track_job_record_metrics + + expect(metric_for_job_by_name(:gauge_migrated_tuples)).to eq(20) + end + + it 'reports the total tuple count for the migration' do + track_job_record_metrics + + expect(metric_for_job_by_name(:gauge_total_tuple_count)).to eq(job_record.batched_migration.total_tuple_count) + end + + it 'reports last updated at timestamp' do + freeze_time do + track_job_record_metrics + + expect(metric_for_job_by_name(:gauge_last_update_time)).to eq(Time.current.to_i) + end + end + + it 'reports summary of query timings' do + summary_labels = labels.merge(operation: 'update_all') + + job_record.metrics['timings']['update_all'].each do |timing| + expect(described_class.metrics[:histogram_timings]).to receive(:observe) + .with(summary_labels, timing) + .and_call_original + end + + track_job_record_metrics + + expect(metric_for_job_by_name(:histogram_timings, job_labels: summary_labels)) + .to eq({ 0.1 => 1.0, 0.25 => 2.0, 0.5 => 3.0, 1 => 4.0, 5 => 5.0 }) + end + + context 'when the tracking record does not having timing metrics' do + before do + job_record.metrics = {} + end + + it 'does not attempt to report query timings' do + summary_labels = labels.merge(operation: 'update_all') + + expect(described_class.metrics[:histogram_timings]).not_to receive(:observe) + + track_job_record_metrics + + expect(metric_for_job_by_name(:histogram_timings, job_labels: summary_labels)) + .to eq({ 0.1 => 0.0, 0.25 => 0.0, 0.5 => 0.0, 1 => 0.0, 5 => 0.0 }) + end + end + + def metric_for_job_by_name(name, job_labels: labels) + described_class.metrics[name].values[job_labels].get + end + end +end diff --git a/spec/lib/gitlab/database/consistency_checker_spec.rb b/spec/lib/gitlab/database/consistency_checker_spec.rb new file mode 100644 index 00000000000..2ff79d20786 --- /dev/null +++ b/spec/lib/gitlab/database/consistency_checker_spec.rb @@ -0,0 +1,189 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::ConsistencyChecker do + let(:batch_size) { 10 } + let(:max_batches) { 4 } + let(:max_runtime) { described_class::MAX_RUNTIME } + let(:metrics_counter) { Gitlab::Metrics.registry.get(:consistency_checks) } + + subject(:consistency_checker) do + described_class.new( + source_model: Namespace, + target_model: Ci::NamespaceMirror, + source_columns: %w[id traversal_ids], + target_columns: %w[namespace_id traversal_ids] + ) + end + + before do + stub_const("#{described_class.name}::BATCH_SIZE", batch_size) + stub_const("#{described_class.name}::MAX_BATCHES", max_batches) + redis_shared_state_cleanup! # For Prometheus Counters + end + + after do + Gitlab::Metrics.reset_registry! + end + + describe '#over_time_limit?' do + before do + allow(consistency_checker).to receive(:start_time).and_return(0) + end + + it 'returns true only if the running time has exceeded MAX_RUNTIME' do + allow(consistency_checker).to receive(:monotonic_time).and_return(0, max_runtime - 1, max_runtime + 1) + expect(consistency_checker.monotonic_time).to eq(0) + expect(consistency_checker.send(:over_time_limit?)).to eq(false) + expect(consistency_checker.send(:over_time_limit?)).to eq(true) + end + end + + describe '#execute' do + context 'when empty tables' do + it 'returns an empty response' do + expected_result = { matches: 0, mismatches: 0, batches: 0, mismatches_details: [], next_start_id: nil } + expect(consistency_checker.execute(start_id: 1)).to eq(expected_result) + end + end + + context 'when the tables contain matching items' do + before do + create_list(:namespace, 50) # This will also create Ci::NameSpaceMirror objects + end + + it 'does not process more than MAX_BATCHES' do + max_batches = 3 + stub_const("#{described_class.name}::MAX_BATCHES", max_batches) + result = consistency_checker.execute(start_id: Namespace.minimum(:id)) + expect(result[:batches]).to eq(max_batches) + expect(result[:matches]).to eq(max_batches * batch_size) + end + + it 'doesn not exceed the MAX_RUNTIME' do + allow(consistency_checker).to receive(:monotonic_time).and_return(0, max_runtime - 1, max_runtime + 1) + result = consistency_checker.execute(start_id: Namespace.minimum(:id)) + expect(result[:batches]).to eq(1) + expect(result[:matches]).to eq(1 * batch_size) + end + + it 'returns the correct number of matches and batches checked' do + expected_result = { + next_start_id: Namespace.minimum(:id) + described_class::MAX_BATCHES * described_class::BATCH_SIZE, + batches: max_batches, + matches: max_batches * batch_size, + mismatches: 0, + mismatches_details: [] + } + expect(consistency_checker.execute(start_id: Namespace.minimum(:id))).to eq(expected_result) + end + + it 'returns the min_id as the next_start_id if the check reaches the last element' do + expect(Gitlab::Metrics).to receive(:counter).at_most(:once) + .with(:consistency_checks, "Consistency Check Results") + .and_call_original + + # Starting from the 5th last element + start_id = Namespace.all.order(id: :desc).limit(5).pluck(:id).last + expected_result = { + next_start_id: Namespace.first.id, + batches: 1, + matches: 5, + mismatches: 0, + mismatches_details: [] + } + expect(consistency_checker.execute(start_id: start_id)).to eq(expected_result) + + expect(metrics_counter.get(source_table: "namespaces", result: "mismatch")).to eq(0) + expect(metrics_counter.get(source_table: "namespaces", result: "match")).to eq(5) + end + end + + context 'when some items are missing from the first table' do + let(:missing_namespace) { Namespace.all.order(:id).limit(2).last } + + before do + create_list(:namespace, 50) # This will also create Ci::NameSpaceMirror objects + missing_namespace.delete + end + + it 'reports the missing elements' do + expected_result = { + next_start_id: Namespace.first.id + described_class::MAX_BATCHES * described_class::BATCH_SIZE, + batches: max_batches, + matches: 39, + mismatches: 1, + mismatches_details: [{ + id: missing_namespace.id, + source_table: nil, + target_table: [missing_namespace.traversal_ids] + }] + } + expect(consistency_checker.execute(start_id: Namespace.first.id)).to eq(expected_result) + + expect(metrics_counter.get(source_table: "namespaces", result: "mismatch")).to eq(1) + expect(metrics_counter.get(source_table: "namespaces", result: "match")).to eq(39) + end + end + + context 'when some items are missing from the second table' do + let(:missing_ci_namespace_mirror) { Ci::NamespaceMirror.all.order(:id).limit(2).last } + + before do + create_list(:namespace, 50) # This will also create Ci::NameSpaceMirror objects + missing_ci_namespace_mirror.delete + end + + it 'reports the missing elements' do + expected_result = { + next_start_id: Namespace.first.id + described_class::MAX_BATCHES * described_class::BATCH_SIZE, + batches: 4, + matches: 39, + mismatches: 1, + mismatches_details: [{ + id: missing_ci_namespace_mirror.namespace_id, + source_table: [missing_ci_namespace_mirror.traversal_ids], + target_table: nil + }] + } + expect(consistency_checker.execute(start_id: Namespace.first.id)).to eq(expected_result) + + expect(metrics_counter.get(source_table: "namespaces", result: "mismatch")).to eq(1) + expect(metrics_counter.get(source_table: "namespaces", result: "match")).to eq(39) + end + end + + context 'when elements are different between the two tables' do + let(:different_namespaces) { Namespace.order(:id).limit(max_batches * batch_size).sample(3).sort_by(&:id) } + + before do + create_list(:namespace, 50) # This will also create Ci::NameSpaceMirror objects + + different_namespaces.each do |namespace| + namespace.update_attribute(:traversal_ids, []) + end + end + + it 'reports the difference between the two tables' do + expected_result = { + next_start_id: Namespace.first.id + described_class::MAX_BATCHES * described_class::BATCH_SIZE, + batches: 4, + matches: 37, + mismatches: 3, + mismatches_details: different_namespaces.map do |namespace| + { + id: namespace.id, + source_table: [[]], + target_table: [[namespace.id]] # old traversal_ids of the namespace + } + end + } + expect(consistency_checker.execute(start_id: Namespace.first.id)).to eq(expected_result) + + expect(metrics_counter.get(source_table: "namespaces", result: "mismatch")).to eq(3) + expect(metrics_counter.get(source_table: "namespaces", result: "match")).to eq(37) + end + end + end +end diff --git a/spec/lib/gitlab/database/each_database_spec.rb b/spec/lib/gitlab/database/each_database_spec.rb index d46c1ca8681..191f7017b4c 100644 --- a/spec/lib/gitlab/database/each_database_spec.rb +++ b/spec/lib/gitlab/database/each_database_spec.rb @@ -58,6 +58,15 @@ RSpec.describe Gitlab::Database::EachDatabase do end end end + + context 'when shared connections are not included' do + it 'only yields the unshared connections' do + expect(Gitlab::Database).to receive(:db_config_share_with).twice.and_return(nil, 'main') + + expect { |b| described_class.each_database_connection(include_shared: false, &b) } + .to yield_successive_args([ActiveRecord::Base.connection, 'main']) + end + end end describe '.each_model_connection' do diff --git a/spec/lib/gitlab/database/load_balancing/setup_spec.rb b/spec/lib/gitlab/database/load_balancing/setup_spec.rb index 4d565ce137a..c44637b8d06 100644 --- a/spec/lib/gitlab/database/load_balancing/setup_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/setup_spec.rb @@ -10,7 +10,6 @@ RSpec.describe Gitlab::Database::LoadBalancing::Setup do expect(setup).to receive(:configure_connection) expect(setup).to receive(:setup_connection_proxy) expect(setup).to receive(:setup_service_discovery) - expect(setup).to receive(:setup_feature_flag_to_model_load_balancing) setup.setup end @@ -120,120 +119,46 @@ RSpec.describe Gitlab::Database::LoadBalancing::Setup do end end - describe '#setup_feature_flag_to_model_load_balancing', :reestablished_active_record_base do + context 'uses correct base models', :reestablished_active_record_base do using RSpec::Parameterized::TableSyntax where do { - "with model LB enabled it picks a dedicated CI connection" => { - env_GITLAB_USE_MODEL_LOAD_BALANCING: 'true', + "it picks a dedicated CI connection" => { env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: nil, request_store_active: false, - ff_use_model_load_balancing: nil, ff_force_no_sharing_primary_model: false, expectations: { main: { read: 'main_replica', write: 'main' }, ci: { read: 'ci_replica', write: 'ci' } } }, - "with model LB enabled and re-use of primary connection it uses CI connection for reads" => { - env_GITLAB_USE_MODEL_LOAD_BALANCING: 'true', + "with re-use of primary connection it uses CI connection for reads" => { env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: 'main', request_store_active: false, - ff_use_model_load_balancing: nil, ff_force_no_sharing_primary_model: false, expectations: { main: { read: 'main_replica', write: 'main' }, ci: { read: 'ci_replica', write: 'main' } } }, - "with model LB disabled it fallbacks to use main" => { - env_GITLAB_USE_MODEL_LOAD_BALANCING: 'false', - env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: nil, - request_store_active: false, - ff_use_model_load_balancing: nil, - ff_force_no_sharing_primary_model: false, - expectations: { - main: { read: 'main_replica', write: 'main' }, - ci: { read: 'main_replica', write: 'main' } - } - }, - "with model LB disabled, but re-use configured it fallbacks to use main" => { - env_GITLAB_USE_MODEL_LOAD_BALANCING: 'false', + "with re-use and FF force_no_sharing_primary_model enabled with RequestStore it sticks FF and uses CI connection for reads and writes" => { env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: 'main', - request_store_active: false, - ff_use_model_load_balancing: nil, - ff_force_no_sharing_primary_model: false, - expectations: { - main: { read: 'main_replica', write: 'main' }, - ci: { read: 'main_replica', write: 'main' } - } - }, - "with FF use_model_load_balancing disabled without RequestStore it uses main" => { - env_GITLAB_USE_MODEL_LOAD_BALANCING: nil, - env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: nil, - request_store_active: false, - ff_use_model_load_balancing: false, - ff_force_no_sharing_primary_model: false, - expectations: { - main: { read: 'main_replica', write: 'main' }, - ci: { read: 'main_replica', write: 'main' } - } - }, - "with FF use_model_load_balancing enabled without RequestStore sticking of FF does not work, so it fallbacks to use main" => { - env_GITLAB_USE_MODEL_LOAD_BALANCING: nil, - env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: nil, - request_store_active: false, - ff_use_model_load_balancing: true, - ff_force_no_sharing_primary_model: false, - expectations: { - main: { read: 'main_replica', write: 'main' }, - ci: { read: 'main_replica', write: 'main' } - } - }, - "with FF use_model_load_balancing disabled with RequestStore it uses main" => { - env_GITLAB_USE_MODEL_LOAD_BALANCING: nil, - env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: nil, - request_store_active: true, - ff_use_model_load_balancing: false, - ff_force_no_sharing_primary_model: false, - expectations: { - main: { read: 'main_replica', write: 'main' }, - ci: { read: 'main_replica', write: 'main' } - } - }, - "with FF use_model_load_balancing enabled with RequestStore it sticks FF and uses CI connection" => { - env_GITLAB_USE_MODEL_LOAD_BALANCING: nil, - env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: nil, request_store_active: true, - ff_use_model_load_balancing: true, - ff_force_no_sharing_primary_model: false, + ff_force_no_sharing_primary_model: true, expectations: { main: { read: 'main_replica', write: 'main' }, ci: { read: 'ci_replica', write: 'ci' } } }, - "with re-use and ff_use_model_load_balancing enabled and FF force_no_sharing_primary_model disabled with RequestStore it sticks FF and uses CI connection for reads" => { - env_GITLAB_USE_MODEL_LOAD_BALANCING: nil, + "with re-use and FF force_no_sharing_primary_model enabled without RequestStore it doesn't use FF and uses CI connection for reads only" => { env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: 'main', request_store_active: true, - ff_use_model_load_balancing: true, ff_force_no_sharing_primary_model: false, expectations: { main: { read: 'main_replica', write: 'main' }, ci: { read: 'ci_replica', write: 'main' } } - }, - "with re-use and ff_use_model_load_balancing enabled and FF force_no_sharing_primary_model enabled with RequestStore it sticks FF and uses CI connection for reads" => { - env_GITLAB_USE_MODEL_LOAD_BALANCING: nil, - env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: 'main', - request_store_active: true, - ff_use_model_load_balancing: true, - ff_force_no_sharing_primary_model: true, - expectations: { - main: { read: 'main_replica', write: 'main' }, - ci: { read: 'ci_replica', write: 'ci' } - } } } end @@ -285,9 +210,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::Setup do end end - stub_env('GITLAB_USE_MODEL_LOAD_BALANCING', env_GITLAB_USE_MODEL_LOAD_BALANCING) stub_env('GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci', env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci) - stub_feature_flags(use_model_load_balancing: ff_use_model_load_balancing) # Make load balancer to force init with a dedicated replicas connections models.each do |_, model| diff --git a/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb b/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb index ad9a3a6e257..e7b5bad8626 100644 --- a/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb @@ -240,7 +240,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a end def software_license_class - Class.new(ActiveRecord::Base) do + Class.new(Gitlab::Database::Migration[2.0]::MigrationRecord) do self.table_name = 'software_licenses' end end @@ -272,7 +272,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a end def ci_instance_variables_class - Class.new(ActiveRecord::Base) do + Class.new(Gitlab::Database::Migration[2.0]::MigrationRecord) do self.table_name = 'ci_instance_variables' end end @@ -303,7 +303,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a end def detached_partitions_class - Class.new(ActiveRecord::Base) do + Class.new(Gitlab::Database::Migration[2.0]::MigrationRecord) do self.table_name = 'detached_partitions' end end @@ -496,11 +496,16 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a Gitlab::Database.database_base_models.each do |db_config_name, model| context "for db_config_name=#{db_config_name}" do around do |example| + verbose_was = ActiveRecord::Migration.verbose + ActiveRecord::Migration.verbose = false + with_reestablished_active_record_base do reconfigure_db_connection(model: ActiveRecord::Base, config_model: model) example.run end + ensure + ActiveRecord::Migration.verbose = verbose_was end before do @@ -543,8 +548,15 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a expect { ignore_error(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas::DDLNotAllowedError) { migration_class.migrate(:down) } }.not_to raise_error when :skipped - expect { migration_class.migrate(:up) }.to raise_error(Gitlab::Database::MigrationHelpers::RestrictGitlabSchema::MigrationSkippedError) - expect { migration_class.migrate(:down) }.to raise_error(Gitlab::Database::MigrationHelpers::RestrictGitlabSchema::MigrationSkippedError) + expect_next_instance_of(migration_class) do |migration_object| + expect(migration_object).to receive(:migration_skipped).and_call_original + expect(migration_object).not_to receive(:up) + expect(migration_object).not_to receive(:down) + expect(migration_object).not_to receive(:change) + end + + migration_class.migrate(:up) + migration_class.migrate(:down) end end end diff --git a/spec/lib/gitlab/database/migration_helpers/v2_spec.rb b/spec/lib/gitlab/database/migration_helpers/v2_spec.rb index acf775b3538..5c054795697 100644 --- a/spec/lib/gitlab/database/migration_helpers/v2_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/v2_spec.rb @@ -96,6 +96,12 @@ RSpec.describe Gitlab::Database::MigrationHelpers::V2 do expect(new_record_1.reload).to have_attributes(status: 1, original: 'updated', renamed: 'updated') expect(new_record_2.reload).to have_attributes(status: 1, original: nil, renamed: nil) end + + it 'requires the helper to run in ddl mode' do + expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_ddl_mode!) + + migration.public_send(operation, :_test_table, :original, :renamed) + end end describe '#rename_column_concurrently' do diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index 9505da8fd12..798eee0de3e 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -1390,6 +1390,8 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end it 'reverses the operations of cleanup_concurrent_column_type_change' do + expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_ddl_mode!) + expect(model).to receive(:check_trigger_permissions!).with(:users) expect(model).to receive(:create_column_from).with( @@ -1415,6 +1417,8 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end it 'passes the type_cast_function, batch_column_name and limit' do + expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_ddl_mode!) + expect(model).to receive(:column_exists?).with(:users, :other_batch_column).and_return(true) expect(model).to receive(:check_trigger_permissions!).with(:users) @@ -2096,7 +2100,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end end - let(:migration_relation) { Gitlab::Database::BackgroundMigration::BatchedMigration.active } + let(:migration_relation) { Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:active) } before do model.initialize_conversion_of_integer_to_bigint(table, columns) @@ -2218,7 +2222,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do subject(:ensure_batched_background_migration_is_finished) { model.ensure_batched_background_migration_is_finished(**configuration) } it 'raises an error when migration exists and is not marked as finished' do - create(:batched_background_migration, configuration.merge(status: :active)) + create(:batched_background_migration, :active, configuration) expect { ensure_batched_background_migration_is_finished } .to raise_error "Expected batched background migration for the given configuration to be marked as 'finished', but it is 'active':" \ @@ -2234,7 +2238,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end it 'does not raise error when migration exists and is marked as finished' do - create(:batched_background_migration, configuration.merge(status: :finished)) + create(:batched_background_migration, :finished, configuration) expect { ensure_batched_background_migration_is_finished } .not_to raise_error @@ -2422,7 +2426,8 @@ RSpec.describe Gitlab::Database::MigrationHelpers do def setup namespace = namespaces.create!(name: 'foo', path: 'foo', type: Namespaces::UserNamespace.sti_name) - projects.create!(namespace_id: namespace.id) + project_namespace = namespaces.create!(name: 'project-foo', path: 'project-foo', type: 'Project', parent_id: namespace.id, visibility_level: 20) + projects.create!(namespace_id: namespace.id, project_namespace_id: project_namespace.id) end it 'generates iids properly for models created after the migration' do diff --git a/spec/lib/gitlab/database/migration_spec.rb b/spec/lib/gitlab/database/migration_spec.rb index 287e738c24e..18bbc6c1dd3 100644 --- a/spec/lib/gitlab/database/migration_spec.rb +++ b/spec/lib/gitlab/database/migration_spec.rb @@ -32,7 +32,7 @@ RSpec.describe Gitlab::Database::Migration do # This breaks upon Rails upgrade. In that case, we'll add a new version in Gitlab::Database::Migration::MIGRATION_CLASSES, # bump .current_version and leave existing migrations and already defined versions of Gitlab::Database::Migration # untouched. - expect(described_class[described_class.current_version].superclass).to eq(ActiveRecord::Migration::Current) + expect(described_class[described_class.current_version]).to be < ActiveRecord::Migration::Current end end diff --git a/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb b/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb index 37efff165c7..f9347a174c4 100644 --- a/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb @@ -75,7 +75,7 @@ RSpec.describe Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers d max_batch_size: 10000, sub_batch_size: 10, job_arguments: %w[], - status: 'active', + status_name: :active, total_tuple_count: pgclass_info.cardinality_estimate) end diff --git a/spec/lib/gitlab/database/migrations/instrumentation_spec.rb b/spec/lib/gitlab/database/migrations/instrumentation_spec.rb index fd8303c379c..c31244060ec 100644 --- a/spec/lib/gitlab/database/migrations/instrumentation_spec.rb +++ b/spec/lib/gitlab/database/migrations/instrumentation_spec.rb @@ -11,6 +11,10 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do describe '#observe' do subject { described_class.new(result_dir: result_dir) } + def load_observation(result_dir, migration_name) + Gitlab::Json.parse(File.read(File.join(result_dir, migration_name, described_class::STATS_FILENAME))) + end + let(:migration_name) { 'test' } let(:migration_version) { '12345' } @@ -87,7 +91,7 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do end context 'retrieving observations' do - subject { instance.observations.first } + subject { load_observation(result_dir, migration_name) } before do observe @@ -98,10 +102,10 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do end it 'records a valid observation', :aggregate_failures do - expect(subject.walltime).not_to be_nil - expect(subject.success).to be_falsey - expect(subject.version).to eq(migration_version) - expect(subject.name).to eq(migration_name) + expect(subject['walltime']).not_to be_nil + expect(subject['success']).to be_falsey + expect(subject['version']).to eq(migration_version) + expect(subject['name']).to eq(migration_name) end end end @@ -113,11 +117,18 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do let(:migration1) { double('migration1', call: nil) } let(:migration2) { double('migration2', call: nil) } + let(:migration_name_2) { 'other_migration' } + let(:migration_version_2) { '98765' } + it 'records observations for all migrations' do subject.observe(version: migration_version, name: migration_name, connection: connection) {} - subject.observe(version: migration_version, name: migration_name, connection: connection) { raise 'something went wrong' } rescue nil + subject.observe(version: migration_version_2, name: migration_name_2, connection: connection) { raise 'something went wrong' } rescue nil + + expect { load_observation(result_dir, migration_name) }.not_to raise_error + expect { load_observation(result_dir, migration_name_2) }.not_to raise_error - expect(subject.observations.size).to eq(2) + # Each observation is a subdirectory of the result_dir, so here we check that we didn't record an extra one + expect(Pathname(result_dir).children.map { |d| d.basename.to_s }).to contain_exactly(migration_name, migration_name_2) end end end diff --git a/spec/lib/gitlab/database/migrations/runner_spec.rb b/spec/lib/gitlab/database/migrations/runner_spec.rb index 84482e6b450..8b1ccf05eb1 100644 --- a/spec/lib/gitlab/database/migrations/runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/runner_spec.rb @@ -124,4 +124,16 @@ RSpec.describe Gitlab::Database::Migrations::Runner do expect(metadata).to match('version' => described_class::SCHEMA_VERSION) end end + + describe '.background_migrations' do + it 'is a TestBackgroundRunner' do + expect(described_class.background_migrations).to be_a(Gitlab::Database::Migrations::TestBackgroundRunner) + end + + it 'is configured with a result dir of /background_migrations' do + runner = described_class.background_migrations + + expect(runner.result_dir).to eq(described_class::BASE_RESULT_DIR.join( 'background_migrations')) + end + end end diff --git a/spec/lib/gitlab/database/migrations/test_background_runner_spec.rb b/spec/lib/gitlab/database/migrations/test_background_runner_spec.rb index c6fe88a7c2d..9407efad91f 100644 --- a/spec/lib/gitlab/database/migrations/test_background_runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/test_background_runner_spec.rb @@ -11,11 +11,17 @@ RSpec.describe Gitlab::Database::Migrations::TestBackgroundRunner, :redis do Sidekiq::Testing.disable! { ex.run } end + let(:result_dir) { Dir.mktmpdir } + + after do + FileUtils.rm_rf(result_dir) + end + context 'without jobs to run' do it 'returns immediately' do - runner = described_class.new + runner = described_class.new(result_dir: result_dir) expect(runner).not_to receive(:run_job) - described_class.new.run_jobs(for_duration: 1.second) + described_class.new(result_dir: result_dir).run_jobs(for_duration: 1.second) end end @@ -30,7 +36,7 @@ RSpec.describe Gitlab::Database::Migrations::TestBackgroundRunner, :redis do context 'finding pending background jobs' do it 'finds all the migrations' do - expect(described_class.new.traditional_background_migrations.to_a.size).to eq(5) + expect(described_class.new(result_dir: result_dir).traditional_background_migrations.to_a.size).to eq(5) end end @@ -53,12 +59,28 @@ RSpec.describe Gitlab::Database::Migrations::TestBackgroundRunner, :redis do end end + def expect_recorded_migration_runs(migrations_to_runs) + migrations_to_runs.each do |migration, runs| + path = File.join(result_dir, migration.name.demodulize) + num_subdirs = Pathname(path).children.count(&:directory?) + expect(num_subdirs).to eq(runs) + end + end + + def expect_migration_runs(migrations_to_run_counts) + expect_migration_call_counts(migrations_to_run_counts) + + yield + + expect_recorded_migration_runs(migrations_to_run_counts) + end + it 'runs the migration class correctly' do calls = [] define_background_migration(migration_name) do |i| calls << i end - described_class.new.run_jobs(for_duration: 1.second) # Any time would work here as we do not advance time + described_class.new(result_dir: result_dir).run_jobs(for_duration: 1.second) # Any time would work here as we do not advance time expect(calls).to contain_exactly(1, 2, 3, 4, 5) end @@ -67,9 +89,9 @@ RSpec.describe Gitlab::Database::Migrations::TestBackgroundRunner, :redis do travel(1.minute) end - expect_migration_call_counts(migration => 3) - - described_class.new.run_jobs(for_duration: 3.minutes) + expect_migration_runs(migration => 3) do + described_class.new(result_dir: result_dir).run_jobs(for_duration: 3.minutes) + end end context 'with multiple migrations to run' do @@ -90,12 +112,12 @@ RSpec.describe Gitlab::Database::Migrations::TestBackgroundRunner, :redis do travel(2.minutes) end - expect_migration_call_counts( + expect_migration_runs( migration => 2, # 1 minute jobs for 90 seconds, can finish the first and start the second other_migration => 1 # 2 minute jobs for 90 seconds, past deadline after a single job - ) - - described_class.new.run_jobs(for_duration: 3.minutes) + ) do + described_class.new(result_dir: result_dir).run_jobs(for_duration: 3.minutes) + end end it 'does not give leftover time to extra migrations' do @@ -107,12 +129,13 @@ RSpec.describe Gitlab::Database::Migrations::TestBackgroundRunner, :redis do other_migration = define_background_migration(other_migration_name) do travel(1.minute) end - expect_migration_call_counts( + + expect_migration_runs( migration => 5, other_migration => 2 - ) - - described_class.new.run_jobs(for_duration: 3.minutes) + ) do + described_class.new(result_dir: result_dir).run_jobs(for_duration: 3.minutes) + end end end end diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb index 4f1d6302331..1026b4370a5 100644 --- a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb +++ b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb @@ -125,6 +125,17 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe expect_table_partitioned_by(partitioned_table, [partition_column]) end + it 'requires the migration helper to be run in DDL mode' do + expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_ddl_mode!) + + migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date + + expect(connection.table_exists?(partitioned_table)).to be(true) + expect(connection.primary_key(partitioned_table)).to eq(new_primary_key) + + expect_table_partitioned_by(partitioned_table, [partition_column]) + end + it 'changes the primary key datatype to bigint' do migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date @@ -191,6 +202,8 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe end it 'creates a partition spanning over each month from the first record' do + expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:with_suppressed).and_yield + migration.partition_table_by_date source_table, partition_column, max_date: max_date expect_range_partitions_for(partitioned_table, { @@ -206,6 +219,8 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe context 'without data' do it 'creates the catchall partition plus two actual partition' do + expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:with_suppressed).and_yield + migration.partition_table_by_date source_table, partition_column, max_date: max_date expect_range_partitions_for(partitioned_table, { @@ -536,6 +551,16 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe migration.finalize_backfilling_partitioned_table source_table end + + it 'requires the migration helper to execute in DML mode' do + expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_dml_mode!) + + expect(Gitlab::BackgroundMigration).to receive(:steal) + .with(described_class::MIGRATION_CLASS_NAME) + .and_yield(background_job) + + migration.finalize_backfilling_partitioned_table source_table + end end context 'when there is missed data' do @@ -627,6 +652,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe allow(backfill).to receive(:perform).and_return(1) end + expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:with_suppressed).and_yield expect(migration).to receive(:disable_statement_timeout).and_call_original expect(migration).to receive(:execute).with("VACUUM FREEZE ANALYZE #{partitioned_table}") diff --git a/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb b/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb index 86e74cf5177..b8c1ecd9089 100644 --- a/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb +++ b/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb @@ -17,7 +17,7 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasMetrics, query_ana process_sql(ActiveRecord::Base, "SELECT 1 FROM projects") end - context 'properly observes all queries', :add_ci_connection do + context 'properly observes all queries', :add_ci_connection, :request_store do using RSpec::Parameterized::TableSyntax where do @@ -28,7 +28,8 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasMetrics, query_ana expectations: { gitlab_schemas: "gitlab_main", db_config_name: "main" - } + }, + setup: nil }, "for query accessing gitlab_ci and gitlab_main" => { model: ApplicationRecord, @@ -36,7 +37,8 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasMetrics, query_ana expectations: { gitlab_schemas: "gitlab_ci,gitlab_main", db_config_name: "main" - } + }, + setup: nil }, "for query accessing gitlab_ci and gitlab_main the gitlab_schemas is always ordered" => { model: ApplicationRecord, @@ -44,7 +46,8 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasMetrics, query_ana expectations: { gitlab_schemas: "gitlab_ci,gitlab_main", db_config_name: "main" - } + }, + setup: nil }, "for query accessing CI database" => { model: Ci::ApplicationRecord, @@ -53,6 +56,62 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasMetrics, query_ana gitlab_schemas: "gitlab_ci", db_config_name: "ci" } + }, + "for query accessing CI database with re-use and disabled sharing" => { + model: Ci::ApplicationRecord, + sql: "SELECT 1 FROM ci_builds", + expectations: { + gitlab_schemas: "gitlab_ci", + db_config_name: "ci", + ci_dedicated_primary_connection: true + }, + setup: ->(_) do + skip_if_multiple_databases_not_setup + stub_env('GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci', 'main') + stub_feature_flags(force_no_sharing_primary_model: true) + end + }, + "for query accessing CI database with re-use and enabled sharing" => { + model: Ci::ApplicationRecord, + sql: "SELECT 1 FROM ci_builds", + expectations: { + gitlab_schemas: "gitlab_ci", + db_config_name: "ci", + ci_dedicated_primary_connection: false + }, + setup: ->(_) do + skip_if_multiple_databases_not_setup + stub_env('GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci', 'main') + stub_feature_flags(force_no_sharing_primary_model: false) + end + }, + "for query accessing CI database without re-use and disabled sharing" => { + model: Ci::ApplicationRecord, + sql: "SELECT 1 FROM ci_builds", + expectations: { + gitlab_schemas: "gitlab_ci", + db_config_name: "ci", + ci_dedicated_primary_connection: true + }, + setup: ->(_) do + skip_if_multiple_databases_not_setup + stub_env('GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci', nil) + stub_feature_flags(force_no_sharing_primary_model: true) + end + }, + "for query accessing CI database without re-use and enabled sharing" => { + model: Ci::ApplicationRecord, + sql: "SELECT 1 FROM ci_builds", + expectations: { + gitlab_schemas: "gitlab_ci", + db_config_name: "ci", + ci_dedicated_primary_connection: true + }, + setup: ->(_) do + skip_if_multiple_databases_not_setup + stub_env('GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci', nil) + stub_feature_flags(force_no_sharing_primary_model: false) + end } } end @@ -63,8 +122,15 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasMetrics, query_ana end it do + stub_env('GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci', nil) + + instance_eval(&setup) if setup + + allow(::Ci::ApplicationRecord.load_balancer).to receive(:configuration) + .and_return(Gitlab::Database::LoadBalancing::Configuration.for_model(::Ci::ApplicationRecord)) + expect(described_class.schemas_metrics).to receive(:increment) - .with(expectations).and_call_original + .with({ ci_dedicated_primary_connection: anything }.merge(expectations)).and_call_original process_sql(model, sql) end diff --git a/spec/lib/gitlab/database/reindexing/grafana_notifier_spec.rb b/spec/lib/gitlab/database/reindexing/grafana_notifier_spec.rb index e76718fe48a..34670696787 100644 --- a/spec/lib/gitlab/database/reindexing/grafana_notifier_spec.rb +++ b/spec/lib/gitlab/database/reindexing/grafana_notifier_spec.rb @@ -74,8 +74,28 @@ RSpec.describe Gitlab::Database::Reindexing::GrafanaNotifier do end describe '#notify_start' do - context 'additional tag is nil' do - subject { described_class.new(api_key, api_url, nil).notify_start(action) } + context 'when Grafana is configured using application settings' do + subject { described_class.new.notify_start(action) } + + let(:payload) do + { + time: (action.action_start.utc.to_f * 1000).to_i, + tags: ['reindex', additional_tag, action.index.tablename, action.index.name], + text: "Started reindexing of #{action.index.name} on #{action.index.tablename}" + } + end + + before do + stub_application_setting(database_grafana_api_key: api_key) + stub_application_setting(database_grafana_api_url: api_url) + stub_application_setting(database_grafana_tag: additional_tag) + end + + it_behaves_like 'interacting with Grafana annotations API' + end + + context 'when there is no additional tag' do + subject { described_class.new(api_key: api_key, api_url: api_url, additional_tag: '').notify_start(action) } let(:payload) do { @@ -88,8 +108,8 @@ RSpec.describe Gitlab::Database::Reindexing::GrafanaNotifier do it_behaves_like 'interacting with Grafana annotations API' end - context 'additional tag is not nil' do - subject { described_class.new(api_key, api_url, additional_tag).notify_start(action) } + context 'additional tag is provided' do + subject { described_class.new(api_key: api_key, api_url: api_url, additional_tag: additional_tag).notify_start(action) } let(:payload) do { @@ -104,8 +124,30 @@ RSpec.describe Gitlab::Database::Reindexing::GrafanaNotifier do end describe '#notify_end' do - context 'additional tag is nil' do - subject { described_class.new(api_key, api_url, nil).notify_end(action) } + context 'when Grafana is configured using application settings' do + subject { described_class.new.notify_end(action) } + + let(:payload) do + { + time: (action.action_start.utc.to_f * 1000).to_i, + tags: ['reindex', additional_tag, action.index.tablename, action.index.name], + text: "Finished reindexing of #{action.index.name} on #{action.index.tablename} (#{action.state})", + timeEnd: (action.action_end.utc.to_f * 1000).to_i, + isRegion: true + } + end + + before do + stub_application_setting(database_grafana_api_key: api_key) + stub_application_setting(database_grafana_api_url: api_url) + stub_application_setting(database_grafana_tag: additional_tag) + end + + it_behaves_like 'interacting with Grafana annotations API' + end + + context 'when there is no additional tag' do + subject { described_class.new(api_key: api_key, api_url: api_url, additional_tag: '').notify_end(action) } let(:payload) do { @@ -120,8 +162,8 @@ RSpec.describe Gitlab::Database::Reindexing::GrafanaNotifier do it_behaves_like 'interacting with Grafana annotations API' end - context 'additional tag is not nil' do - subject { described_class.new(api_key, api_url, additional_tag).notify_end(action) } + context 'additional tag is provided' do + subject { described_class.new(api_key: api_key, api_url: api_url, additional_tag: additional_tag).notify_end(action) } let(:payload) do { diff --git a/spec/lib/gitlab/database/schema_cache_with_renamed_table_spec.rb b/spec/lib/gitlab/database/schema_cache_with_renamed_table_spec.rb index 7caee414719..0bea348e6b4 100644 --- a/spec/lib/gitlab/database/schema_cache_with_renamed_table_spec.rb +++ b/spec/lib/gitlab/database/schema_cache_with_renamed_table_spec.rb @@ -68,8 +68,8 @@ RSpec.describe Gitlab::Database::SchemaCacheWithRenamedTable do describe 'when the table behind a model is actually a view' do let(:group) { create(:group) } - let(:project_attributes) { attributes_for(:project, namespace_id: group.id).except(:creator) } - let(:record) { old_model.create!(project_attributes) } + let(:attrs) { attributes_for(:project, namespace_id: group.id, project_namespace_id: group.id).except(:creator) } + let(:record) { old_model.create!(attrs) } it 'can persist records' do expect(record.reload.attributes).to eq(new_model.find(record.id).attributes) |