diff options
Diffstat (limited to 'lib/gitlab/database')
27 files changed, 277 insertions, 259 deletions
diff --git a/lib/gitlab/database/background_migration/batch_optimizer.rb b/lib/gitlab/database/background_migration/batch_optimizer.rb index 0668490dda8..58c4a214077 100644 --- a/lib/gitlab/database/background_migration/batch_optimizer.rb +++ b/lib/gitlab/database/background_migration/batch_optimizer.rb @@ -20,7 +20,8 @@ module Gitlab TARGET_EFFICIENCY = (0.9..0.95).freeze # Lower and upper bound for the batch size - ALLOWED_BATCH_SIZE = (1_000..2_000_000).freeze + MIN_BATCH_SIZE = 1_000 + MAX_BATCH_SIZE = 2_000_000 # Limit for the multiplier of the batch size MAX_MULTIPLIER = 1.2 @@ -43,7 +44,8 @@ module Gitlab return unless Feature.enabled?(:optimize_batched_migrations, type: :ops, default_enabled: :yaml) if multiplier = batch_size_multiplier - migration.batch_size = (migration.batch_size * multiplier).to_i.clamp(ALLOWED_BATCH_SIZE) + max_batch = migration.max_batch_size || MAX_BATCH_SIZE + migration.batch_size = (migration.batch_size * multiplier).to_i.clamp(MIN_BATCH_SIZE, max_batch) migration.save! end end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 290fa51692a..185b6d9629f 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -12,22 +12,58 @@ module Gitlab MAX_ATTEMPTS = 3 STUCK_JOBS_TIMEOUT = 1.hour.freeze - enum status: { - pending: 0, - running: 1, - failed: 2, - succeeded: 3 - } - belongs_to :batched_migration, foreign_key: :batched_background_migration_id + has_many :batched_job_transition_logs, foreign_key: :batched_background_migration_job_id - scope :active, -> { where(status: [:pending, :running]) } + scope :active, -> { with_statuses(:pending, :running) } scope :stuck, -> { active.where('updated_at <= ?', STUCK_JOBS_TIMEOUT.ago) } - scope :retriable, -> { from_union([failed.where('attempts < ?', MAX_ATTEMPTS), self.stuck]) } - scope :except_succeeded, -> { where(status: self.statuses.except(:succeeded).values) } - scope :successful_in_execution_order, -> { where.not(finished_at: nil).succeeded.order(:finished_at) } + scope :retriable, -> { from_union([with_status(:failed).where('attempts < ?', MAX_ATTEMPTS), self.stuck]) } + scope :except_succeeded, -> { without_status(:succeeded) } + scope :successful_in_execution_order, -> { where.not(finished_at: nil).with_status(:succeeded).order(:finished_at) } scope :with_preloads, -> { preload(:batched_migration) } + state_machine :status, initial: :pending do + state :pending, value: 0 + state :running, value: 1 + state :failed, value: 2 + state :succeeded, value: 3 + + event :succeed do + transition any => :succeeded + end + + event :failure do + transition any => :failed + end + + event :run do + transition any => :running + end + + before_transition any => [:failed, :succeeded] do |job| + job.finished_at = Time.current + end + + before_transition any => :running do |job| + job.attempts += 1 + job.started_at = Time.current + job.finished_at = nil + job.metrics = {} + end + + after_transition do |job, transition| + error_hash = transition.args.find { |arg| arg[:error].present? } + + exception = error_hash&.fetch(:error) + + job.batched_job_transition_logs.create(previous_status: transition.from, next_status: transition.to, exception_class: exception&.class, exception_message: exception&.message) + + Gitlab::ErrorTracking.track_exception(exception, batched_job_id: job.id) if exception + + Gitlab::AppLogger.info(message: 'BatchedJob transition', batched_job_id: job.id, previous_state: transition.from_name, new_state: transition.to_name) + end + end + delegate :job_class, :table_name, :column_name, :job_arguments, to: :batched_migration, prefix: :migration diff --git a/lib/gitlab/database/background_migration/batched_job_transition_log.rb b/lib/gitlab/database/background_migration/batched_job_transition_log.rb new file mode 100644 index 00000000000..418bf1a101f --- /dev/null +++ b/lib/gitlab/database/background_migration/batched_job_transition_log.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class BatchedJobTransitionLog < ApplicationRecord + include PartitionedTable + + self.table_name = :batched_background_migration_job_transition_logs + + self.primary_key = :id + + partitioned_by :created_at, strategy: :monthly, retain_for: 6.months + + belongs_to :batched_job, foreign_key: :batched_background_migration_job_id + + validates :previous_status, :next_status, :batched_job, presence: true + + validates :exception_class, length: { maximum: 100 } + validates :exception_message, length: { maximum: 1000 } + + enum previous_status: Gitlab::Database::BackgroundMigration::BatchedJob.state_machine.states.map(&:name), _prefix: true + enum next_status: Gitlab::Database::BackgroundMigration::BatchedJob.state_machine.states.map(&:name), _prefix: true + end + end + end +end diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 2f066039874..1f8ca982ed5 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -47,7 +47,7 @@ module Gitlab def self.successful_rows_counts(migrations) BatchedJob - .succeeded + .with_status(:succeeded) .where(batched_background_migration_id: migrations) .group(:batched_background_migration_id) .sum(:batch_size) @@ -71,7 +71,7 @@ module Gitlab end def retry_failed_jobs! - batched_jobs.failed.each_batch(of: 100) do |batch| + batched_jobs.with_status(:failed).each_batch(of: 100) do |batch| self.class.transaction do batch.lock.each(&:split_and_retry!) self.active! @@ -102,7 +102,7 @@ module Gitlab end def migrated_tuple_count - batched_jobs.succeeded.sum(:batch_size) + batched_jobs.with_status(:succeeded).sum(:batch_size) end def prometheus_labels diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 14e3919986e..9308bae20cf 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -67,7 +67,7 @@ module Gitlab Gitlab::AppLogger.warn "Batched background migration for the given configuration is already finished: #{configuration}" else migration.finalizing! - migration.batched_jobs.pending.each { |job| migration_wrapper.perform(job) } + migration.batched_jobs.with_status(:pending).each { |job| migration_wrapper.perform(job) } run_migration_while(migration, :finalizing) @@ -95,7 +95,8 @@ module Gitlab active_migration.table_name, active_migration.column_name, batch_min_value: batch_min_value, - batch_size: active_migration.batch_size) + batch_size: active_migration.batch_size, + job_arguments: active_migration.job_arguments) return if next_batch_bounds.nil? @@ -115,7 +116,7 @@ module Gitlab def finish_active_migration(active_migration) return if active_migration.batched_jobs.active.exists? - if active_migration.batched_jobs.failed.exists? + if active_migration.batched_jobs.with_status(:failed).exists? active_migration.failed! else active_migration.finished! diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb index e37df102872..057f856d859 100644 --- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -6,6 +6,10 @@ module Gitlab class BatchedMigrationWrapper extend Gitlab::Utils::StrongMemoize + def initialize(connection: ApplicationRecord.connection) + @connection = connection + end + # Wraps the execution of a batched_background_migration. # # Updates the job's tracking records with the status of the migration @@ -18,24 +22,25 @@ module Gitlab execute_batch(batch_tracking_record) - batch_tracking_record.status = :succeeded - rescue Exception # rubocop:disable Lint/RescueException - batch_tracking_record.status = :failed + batch_tracking_record.succeed! + rescue Exception => error # rubocop:disable Lint/RescueException + batch_tracking_record.failure!(error: error) raise ensure - finish_tracking_execution(batch_tracking_record) track_prometheus_metrics(batch_tracking_record) end private + attr_reader :connection + def start_tracking_execution(tracking_record) - tracking_record.update!(attempts: tracking_record.attempts + 1, status: :running, started_at: Time.current, finished_at: nil, metrics: {}) + tracking_record.run! end def execute_batch(tracking_record) - job_instance = tracking_record.migration_job_class.new + job_instance = migration_instance_for(tracking_record.migration_job_class) job_instance.perform( tracking_record.min_value, @@ -51,9 +56,12 @@ module Gitlab end end - def finish_tracking_execution(tracking_record) - tracking_record.finished_at = Time.current - tracking_record.save! + def migration_instance_for(job_class) + if job_class < Gitlab::BackgroundMigration::BaseJob + job_class.new(connection: connection) + else + job_class.new + end end def track_prometheus_metrics(tracking_record) diff --git a/lib/gitlab/database/dynamic_model_helpers.rb b/lib/gitlab/database/dynamic_model_helpers.rb index 220062f1bc6..ad7dea8f0d9 100644 --- a/lib/gitlab/database/dynamic_model_helpers.rb +++ b/lib/gitlab/database/dynamic_model_helpers.rb @@ -5,16 +5,19 @@ module Gitlab module DynamicModelHelpers BATCH_SIZE = 1_000 - def define_batchable_model(table_name) - Class.new(ActiveRecord::Base) do + def define_batchable_model(table_name, connection:) + klass = Class.new(ActiveRecord::Base) do include EachBatch self.table_name = table_name self.inheritance_column = :_type_disabled end + + klass.connection = connection + klass end - def each_batch(table_name, scope: ->(table) { table.all }, of: BATCH_SIZE) + def each_batch(table_name, connection:, scope: ->(table) { table.all }, of: BATCH_SIZE) if transaction_open? raise <<~MSG.squish each_batch should not run inside a transaction, you can disable @@ -23,12 +26,12 @@ module Gitlab MSG end - scope.call(define_batchable_model(table_name)) + scope.call(define_batchable_model(table_name, connection: connection)) .each_batch(of: of) { |batch| yield batch } end - def each_batch_range(table_name, scope: ->(table) { table.all }, of: BATCH_SIZE) - each_batch(table_name, scope: scope, of: of) do |batch| + def each_batch_range(table_name, connection:, scope: ->(table) { table.all }, of: BATCH_SIZE) + each_batch(table_name, connection: connection, scope: scope, of: of) do |batch| yield batch.pluck('MIN(id), MAX(id)').first end end diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb index 7c9e65e6691..c3eea0515d4 100644 --- a/lib/gitlab/database/each_database.rb +++ b/lib/gitlab/database/each_database.rb @@ -14,18 +14,40 @@ module Gitlab end end - def each_model_connection(models) + def each_model_connection(models, &blk) models.each do |model| - connection_name = model.connection.pool.db_config.name - - with_shared_connection(model.connection, connection_name) do - yield model, connection_name + # If model is shared, iterate all available base connections + # Example: `LooseForeignKeys::DeletedRecord` + if model < ::Gitlab::Database::SharedModel + with_shared_model_connections(model, &blk) + else + with_model_connection(model, &blk) end end end private + def with_shared_model_connections(shared_model, &blk) + Gitlab::Database.database_base_models.each_pair do |connection_name, connection_model| + if shared_model.limit_connection_names + next unless shared_model.limit_connection_names.include?(connection_name.to_sym) + end + + with_shared_connection(connection_model.connection, connection_name) do + yield shared_model, connection_name + end + end + end + + def with_model_connection(model, &blk) + connection_name = model.connection.pool.db_config.name + + with_shared_connection(model.connection, connection_name) do + yield model, connection_name + end + end + def with_shared_connection(connection, connection_name) Gitlab::Database::SharedModel.using_connection(connection) do Gitlab::AppLogger.debug(message: 'Switched database connection', connection_name: connection_name) diff --git a/lib/gitlab/database/gitlab_loose_foreign_keys.yml b/lib/gitlab/database/gitlab_loose_foreign_keys.yml deleted file mode 100644 index d694165574d..00000000000 --- a/lib/gitlab/database/gitlab_loose_foreign_keys.yml +++ /dev/null @@ -1,160 +0,0 @@ ---- -dast_site_profiles_pipelines: - - table: ci_pipelines - column: ci_pipeline_id - on_delete: async_delete -vulnerability_feedback: - - table: ci_pipelines - column: pipeline_id - on_delete: async_nullify -ci_pipeline_chat_data: - - table: chat_names - column: chat_name_id - on_delete: async_delete -dast_scanner_profiles_builds: - - table: ci_builds - column: ci_build_id - on_delete: async_delete -dast_site_profiles_builds: - - table: ci_builds - column: ci_build_id - on_delete: async_delete -dast_profiles_pipelines: - - table: ci_pipelines - column: ci_pipeline_id - on_delete: async_delete -clusters_applications_runners: - - table: ci_runners - column: runner_id - on_delete: async_nullify -ci_job_token_project_scope_links: - - table: users - column: added_by_id - on_delete: async_nullify -ci_daily_build_group_report_results: - - table: namespaces - column: group_id - on_delete: async_delete - - table: projects - column: project_id - on_delete: async_delete -ci_freeze_periods: - - table: projects - column: project_id - on_delete: async_delete -ci_pending_builds: - - table: namespaces - column: namespace_id - on_delete: async_delete - - table: projects - column: project_id - on_delete: async_delete -ci_resource_groups: - - table: projects - column: project_id - on_delete: async_delete -ci_runner_namespaces: - - table: namespaces - column: namespace_id - on_delete: async_delete -ci_running_builds: - - table: projects - column: project_id - on_delete: async_delete -ci_namespace_mirrors: - - table: namespaces - column: namespace_id - on_delete: async_delete -ci_build_report_results: - - table: projects - column: project_id - on_delete: async_delete -ci_builds: - - table: users - column: user_id - on_delete: async_nullify -ci_pipelines: - - table: merge_requests - column: merge_request_id - on_delete: async_delete - - table: external_pull_requests - column: external_pull_request_id - on_delete: async_nullify - - table: users - column: user_id - on_delete: async_nullify -ci_project_mirrors: - - table: projects - column: project_id - on_delete: async_delete - - table: namespaces - column: namespace_id - on_delete: async_delete -ci_unit_tests: - - table: projects - column: project_id - on_delete: async_delete -merge_requests: - - table: ci_pipelines - column: head_pipeline_id - on_delete: async_nullify -vulnerability_statistics: - - table: ci_pipelines - column: latest_pipeline_id - on_delete: async_nullify -vulnerability_occurrence_pipelines: - - table: ci_pipelines - column: pipeline_id - on_delete: async_delete -packages_build_infos: - - table: ci_pipelines - column: pipeline_id - on_delete: async_nullify -packages_package_file_build_infos: - - table: ci_pipelines - column: pipeline_id - on_delete: async_nullify -pages_deployments: - - table: ci_builds - column: ci_build_id - on_delete: async_nullify -terraform_state_versions: - - table: ci_builds - column: ci_build_id - on_delete: async_nullify -merge_request_metrics: - - table: ci_pipelines - column: pipeline_id - on_delete: async_delete -project_pages_metadata: - - table: ci_job_artifacts - column: artifacts_archive_id - on_delete: async_nullify -ci_pipeline_schedules: - - table: users - column: owner_id - on_delete: async_nullify -ci_group_variables: - - table: namespaces - column: group_id - on_delete: async_delete -ci_minutes_additional_packs: - - table: namespaces - column: namespace_id - on_delete: async_delete -requirements_management_test_reports: - - table: ci_builds - column: build_id - on_delete: async_nullify -security_scans: - - table: ci_builds - column: build_id - on_delete: async_delete -ci_secure_files: - - table: projects - column: project_id - on_delete: async_delete -ci_pipeline_artifacts: - - table: projects - column: project_id - on_delete: async_delete diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 14807494a79..7adf6ba6afb 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -78,7 +78,11 @@ module Gitlab # All tables from `information_schema.` are `:gitlab_shared` return :gitlab_shared if schema_name == 'information_schema' - # All tables that start with `_test_` are shared and ignored + return :gitlab_main if table_name.start_with?('_test_gitlab_main_') + + return :gitlab_ci if table_name.start_with?('_test_gitlab_ci_') + + # All tables that start with `_test_` without a following schema are shared and ignored return :gitlab_shared if table_name.start_with?('_test_') # All `pg_` tables are marked as `shared` diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index fb5d8cfa32f..93cd75ce5a7 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -4,6 +4,7 @@ agent_group_authorizations: :gitlab_main agent_project_authorizations: :gitlab_main alert_management_alert_assignees: :gitlab_main alert_management_alerts: :gitlab_main +alert_management_alert_metric_images: :gitlab_main alert_management_alert_user_mentions: :gitlab_main alert_management_http_integrations: :gitlab_main allowed_email_domains: :gitlab_main @@ -84,6 +85,7 @@ ci_instance_variables: :gitlab_ci ci_job_artifacts: :gitlab_ci ci_job_token_project_scope_links: :gitlab_ci ci_job_variables: :gitlab_ci +ci_job_artifact_states: :gitlab_ci ci_minutes_additional_packs: :gitlab_ci ci_namespace_monthly_usages: :gitlab_ci ci_namespace_mirrors: :gitlab_ci @@ -552,3 +554,4 @@ x509_commit_signatures: :gitlab_main x509_issuers: :gitlab_main zentao_tracker_data: :gitlab_main zoom_meetings: :gitlab_main +batched_background_migration_job_transition_logs: :gitlab_main diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index e769cb5c35c..63444ebe169 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -74,11 +74,24 @@ module Gitlab # With connection re-use the primary connection can be overwritten # to be used from different model def primary_connection_specification_name - (@primary_model || @model).connection_specification_name + primary_model_or_model_if_enabled.connection_specification_name end - def primary_db_config - (@primary_model || @model).connection_db_config + def primary_model_or_model_if_enabled + if force_no_sharing_primary_model? + @model + else + @primary_model || @model + end + end + + def force_no_sharing_primary_model? + return false unless @primary_model # Doesn't matter since we don't have an overriding primary model + return false unless ::Gitlab::SafeRequestStore.active? + + ::Gitlab::SafeRequestStore.fetch(:force_no_sharing_primary_model) do + ::Feature::FlipperFeature.table_exists? && ::Feature.enabled?(:force_no_sharing_primary_model, default_enabled: :yaml) + end end def replica_db_config diff --git a/lib/gitlab/database/loose_foreign_keys.rb b/lib/gitlab/database/loose_foreign_keys.rb index 1ecfb5ce47f..1338b18a099 100644 --- a/lib/gitlab/database/loose_foreign_keys.rb +++ b/lib/gitlab/database/loose_foreign_keys.rb @@ -28,7 +28,11 @@ module Gitlab end def self.loose_foreign_keys_yaml - @loose_foreign_keys_yaml ||= YAML.load_file(Rails.root.join('lib/gitlab/database/gitlab_loose_foreign_keys.yml')) + @loose_foreign_keys_yaml ||= YAML.load_file(self.loose_foreign_keys_yaml_path) + end + + def self.loose_foreign_keys_yaml_path + @loose_foreign_keys_yaml_path ||= Rails.root.join('config/gitlab_loose_foreign_keys.yml') end private_class_method :build_definition diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index aa5ac1e3486..63c031a6d0b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -9,6 +9,18 @@ module Gitlab include RenameTableHelpers include AsyncIndexes::MigrationHelpers + def define_batchable_model(table_name, connection: self.connection) + super(table_name, connection: connection) + end + + def each_batch(table_name, connection: self.connection, **kwargs) + super(table_name, connection: connection, **kwargs) + end + + def each_batch_range(table_name, connection: self.connection, **kwargs) + super(table_name, connection: connection, **kwargs) + end + # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS MAX_IDENTIFIER_NAME_LENGTH = 63 DEFAULT_TIMESTAMP_COLUMNS = %i[created_at updated_at].freeze @@ -429,6 +441,7 @@ module Gitlab def with_lock_retries(*args, **kwargs, &block) raise_on_exhaustion = !!kwargs.delete(:raise_on_exhaustion) merged_args = { + connection: connection, klass: self.class, logger: Gitlab::BackgroundMigration::Logger, allow_savepoints: true @@ -1054,9 +1067,18 @@ module Gitlab Arel::Nodes::SqlLiteral.new(replace.to_sql) end - def remove_foreign_key_if_exists(...) - if foreign_key_exists?(...) - remove_foreign_key(...) + def remove_foreign_key_if_exists(source, target = nil, **kwargs) + reverse_lock_order = kwargs.delete(:reverse_lock_order) + return unless foreign_key_exists?(source, target, **kwargs) + + if target && reverse_lock_order && transaction_open? + execute("LOCK TABLE #{target}, #{source} IN ACCESS EXCLUSIVE MODE") + end + + if target + remove_foreign_key(source, target, **kwargs) + else + remove_foreign_key(source, **kwargs) end end diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index dcaf7fad05f..a2a4a37ab87 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -66,6 +66,7 @@ module Gitlab batch_max_value: nil, batch_class_name: BATCH_CLASS_NAME, batch_size: BATCH_SIZE, + max_batch_size: nil, sub_batch_size: SUB_BATCH_SIZE ) @@ -86,7 +87,7 @@ module Gitlab migration_status = batch_max_value.nil? ? :finished : :active batch_max_value ||= batch_min_value - migration = Gitlab::Database::BackgroundMigration::BatchedMigration.create!( + migration = Gitlab::Database::BackgroundMigration::BatchedMigration.new( job_class_name: job_class_name, table_name: batch_table_name, column_name: batch_column_name, @@ -97,19 +98,28 @@ module Gitlab batch_class_name: batch_class_name, batch_size: batch_size, sub_batch_size: sub_batch_size, - status: migration_status) + status: migration_status + ) - # This guard is necessary since #total_tuple_count was only introduced schema-wise, - # after this migration helper had been used for the first time. - return migration unless migration.respond_to?(:total_tuple_count) + # Below `BatchedMigration` attributes were introduced after the + # initial `batched_background_migrations` table was created, so any + # migrations that ran relying on initial table schema would not know + # about columns introduced later on because this model is not + # isolated in migrations, which is why we need to check for existence + # of these columns first. + if migration.respond_to?(:max_batch_size) + migration.max_batch_size = max_batch_size + end - # We keep track of the estimated number of tuples to reason later - # about the overall progress of a migration. - migration.total_tuple_count = Gitlab::Database::SharedModel.using_connection(connection) do - Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate + if migration.respond_to?(:total_tuple_count) + # We keep track of the estimated number of tuples to reason later + # about the overall progress of a migration. + migration.total_tuple_count = Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate + end end - migration.save! + migration.save! migration end end diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index 1f7e81cae84..7f34768350b 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -15,30 +15,26 @@ module Gitlab end def observe(version:, name:, connection:, &block) - observation = Observation.new(version, name) - observation.success = true + observation = Observation.new(version: version, name: name, success: false) observers = observer_classes.map { |c| c.new(observation, @result_dir, connection) } - exception = nil - on_each_observer(observers) { |observer| observer.before } - observation.walltime = Benchmark.realtime do - yield - rescue StandardError => e - exception = e - observation.success = false - end + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + yield + + observation.success = true + + observation + ensure + observation.walltime = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start on_each_observer(observers) { |observer| observer.after } on_each_observer(observers) { |observer| observer.record } record_observation(observation) - - raise exception if exception - - observation end private diff --git a/lib/gitlab/database/migrations/lock_retry_mixin.rb b/lib/gitlab/database/migrations/lock_retry_mixin.rb index fff0f35e33c..9774797676a 100644 --- a/lib/gitlab/database/migrations/lock_retry_mixin.rb +++ b/lib/gitlab/database/migrations/lock_retry_mixin.rb @@ -9,6 +9,10 @@ module Gitlab migration.class end + def migration_connection + migration.connection + end + def enable_lock_retries? # regular AR migrations don't have this, # only ones inheriting from Gitlab::Database::Migration have @@ -24,6 +28,7 @@ module Gitlab def ddl_transaction(migration, &block) if use_transaction?(migration) && migration.enable_lock_retries? Gitlab::Database::WithLockRetries.new( + connection: migration.migration_connection, klass: migration.migration_class, logger: Gitlab::BackgroundMigration::Logger ).run(raise_on_exhaustion: false, &block) diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb index a494c357950..228eea3393c 100644 --- a/lib/gitlab/database/migrations/observation.rb +++ b/lib/gitlab/database/migrations/observation.rb @@ -10,7 +10,8 @@ module Gitlab :walltime, :success, :total_database_size_change, - :query_statistics + :query_statistics, + keyword_init: true ) end end diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb index 1343354715a..c7d8bdf30bc 100644 --- a/lib/gitlab/database/partitioning.rb +++ b/lib/gitlab/database/partitioning.rb @@ -3,19 +3,8 @@ module Gitlab module Database module Partitioning - class TableWithoutModel - include PartitionedTable::ClassMethods - - attr_reader :table_name - - def initialize(table_name:, partitioned_column:, strategy:) - @table_name = table_name - partitioned_by(partitioned_column, strategy: strategy) - end - - def connection - Gitlab::Database::SharedModel.connection - end + class TableWithoutModel < Gitlab::Database::SharedModel + include PartitionedTable end class << self @@ -77,7 +66,15 @@ module Gitlab def registered_for_sync registered_models + registered_tables.map do |table| - TableWithoutModel.new(**table) + table_without_model(**table) + end + end + + def table_without_model(table_name:, partitioned_column:, strategy:, limit_connection_names: nil) + Class.new(TableWithoutModel).tap do |klass| + klass.table_name = table_name + klass.partitioned_by(partitioned_column, strategy: strategy) + klass.limit_connection_names = limit_connection_names end end end diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index ba6fa0cf278..ab414f91169 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -12,10 +12,15 @@ module Gitlab def initialize(model) @model = model + @connection_name = model.connection.pool.db_config.name end def sync_partitions - Gitlab::AppLogger.info(message: "Checking state of dynamic postgres partitions", table_name: model.table_name) + Gitlab::AppLogger.info( + message: "Checking state of dynamic postgres partitions", + table_name: model.table_name, + connection_name: @connection_name + ) # Double-checking before getting the lease: # The prevailing situation is no missing partitions and no extra partitions @@ -29,10 +34,13 @@ module Gitlab detach(partitions_to_detach) unless partitions_to_detach.empty? end rescue StandardError => e - Gitlab::AppLogger.error(message: "Failed to create / detach partition(s)", - table_name: model.table_name, - exception_class: e.class, - exception_message: e.message) + Gitlab::AppLogger.error( + message: "Failed to create / detach partition(s)", + table_name: model.table_name, + exception_class: e.class, + exception_message: e.message, + connection_name: @connection_name + ) end private @@ -98,9 +106,12 @@ module Gitlab Postgresql::DetachedPartition.create!(table_name: partition.partition_name, drop_after: RETAIN_DETACHED_PARTITIONS_FOR.from_now) - Gitlab::AppLogger.info(message: "Detached Partition", - partition_name: partition.partition_name, - table_name: partition.table) + Gitlab::AppLogger.info( + message: "Detached Partition", + partition_name: partition.partition_name, + table_name: partition.table, + connection_name: @connection_name + ) end def assert_partition_detachable!(partition) diff --git a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb index f551fa06cad..9cab2c51b3f 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb @@ -49,7 +49,7 @@ module Gitlab end def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table) + define_batchable_model(source_table, connection: connection) .where(source_key_column => start_id..stop_id) end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 984c708aa48..e56ffddac4f 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -5,7 +5,6 @@ module Gitlab module PartitioningMigrationHelpers module TableManagementHelpers include ::Gitlab::Database::SchemaHelpers - include ::Gitlab::Database::DynamicModelHelpers include ::Gitlab::Database::MigrationHelpers include ::Gitlab::Database::Migrations::BackgroundMigrationHelpers diff --git a/lib/gitlab/database/query_analyzers/base.rb b/lib/gitlab/database/query_analyzers/base.rb index 0802d3c8013..5f321ece962 100644 --- a/lib/gitlab/database/query_analyzers/base.rb +++ b/lib/gitlab/database/query_analyzers/base.rb @@ -48,11 +48,15 @@ module Gitlab end def self.context_key - "#{self.class.name}_context" + @context_key ||= "analyzer_#{self.analyzer_key}_context".to_sym end def self.suppress_key - "#{self.class.name}_suppressed" + @suppress_key ||= "analyzer_#{self.analyzer_key}_suppressed".to_sym + end + + def self.analyzer_key + @analyzer_key ||= self.name.demodulize.underscore.to_sym end end end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index 2e3db2a5c6e..a604f79dc41 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -56,6 +56,9 @@ module Gitlab context[:transaction_depth_by_db][database] -= 1 if context[:transaction_depth_by_db][database] == 0 context[:modified_tables_by_db][database].clear + + # Attempt to troubleshoot https://gitlab.com/gitlab-org/gitlab/-/issues/351531 + ::CrossDatabaseModification::TransactionStackTrackRecord.log_gitlab_transactions_stack(action: :end_of_transaction) elsif context[:transaction_depth_by_db][database] < 0 context[:transaction_depth_by_db][database] = 0 raise CrossDatabaseModificationAcrossUnsupportedTablesError, "Misaligned cross-DB transactions discovered at query #{sql}. This could be a bug in #{self.class} or a valid issue to investigate. Read more at https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions ." @@ -87,6 +90,8 @@ module Gitlab all_tables = context[:modified_tables_by_db].values.map(&:to_a).flatten schemas = ::Gitlab::Database::GitlabSchema.table_schemas(all_tables) + schemas += ApplicationRecord.gitlab_transactions_stack + if schemas.many? message = "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \ "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables." \ diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb index 9ddc5391689..f96de13006f 100644 --- a/lib/gitlab/database/schema_helpers.rb +++ b/lib/gitlab/database/schema_helpers.rb @@ -73,6 +73,7 @@ module Gitlab def with_lock_retries(&block) Gitlab::Database::WithLockRetries.new( + connection: connection, klass: self.class, logger: Gitlab::BackgroundMigration::Logger ).run(&block) diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb index 17d7886e8c8..563fab692ef 100644 --- a/lib/gitlab/database/shared_model.rb +++ b/lib/gitlab/database/shared_model.rb @@ -6,6 +6,10 @@ module Gitlab class SharedModel < ActiveRecord::Base self.abstract_class = true + # if shared model is used, this allows to limit connections + # on which this model is being shared + class_attribute :limit_connection_names, default: nil + class << self def using_connection(connection) previous_connection = self.overriding_connection diff --git a/lib/gitlab/database/with_lock_retries.rb b/lib/gitlab/database/with_lock_retries.rb index f9d467ae5cc..f2c5bb9088f 100644 --- a/lib/gitlab/database/with_lock_retries.rb +++ b/lib/gitlab/database/with_lock_retries.rb @@ -61,7 +61,7 @@ module Gitlab [10.seconds, 10.minutes] ].freeze - def initialize(logger: NULL_LOGGER, allow_savepoints: true, timing_configuration: DEFAULT_TIMING_CONFIGURATION, klass: nil, env: ENV, connection: ActiveRecord::Base.connection) + def initialize(logger: NULL_LOGGER, allow_savepoints: true, timing_configuration: DEFAULT_TIMING_CONFIGURATION, klass: nil, env: ENV, connection:) @logger = logger @klass = klass @allow_savepoints = allow_savepoints |