diff options
Diffstat (limited to 'lib/gitlab/database/migration_helpers.rb')
| -rw-r--r-- | lib/gitlab/database/migration_helpers.rb | 158 |
1 files changed, 18 insertions, 140 deletions
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index fd09c31e994..006a24da8fe 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -3,10 +3,10 @@ module Gitlab module Database module MigrationHelpers + include Migrations::BackgroundMigrationHelpers + # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS MAX_IDENTIFIER_NAME_LENGTH = 63 - BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job - BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time PERMITTED_TIMESTAMP_COLUMNS = %i[created_at updated_at deleted_at].to_set.freeze DEFAULT_TIMESTAMP_COLUMNS = %i[created_at updated_at].freeze @@ -136,6 +136,10 @@ module Gitlab 'in the body of your migration class' end + index_name = index_name[:name] if index_name.is_a?(Hash) + + raise 'remove_concurrent_index_by_name must get an index name as the second argument' if index_name.blank? + options = options.merge({ algorithm: :concurrently }) unless index_exists_by_name?(table_name, index_name) @@ -477,7 +481,7 @@ module Gitlab # type is used. # batch_column_name - option is for tables without primary key, in this # case another unique integer column can be used. Example: :user_id - def rename_column_concurrently(table, old, new, type: nil, batch_column_name: :id) + def rename_column_concurrently(table, old, new, type: nil, type_cast_function: nil, batch_column_name: :id) unless column_exists?(table, batch_column_name) raise "Column #{batch_column_name} does not exist on #{table}" end @@ -488,7 +492,7 @@ module Gitlab check_trigger_permissions!(table) - create_column_from(table, old, new, type: type, batch_column_name: batch_column_name) + create_column_from(table, old, new, type: type, batch_column_name: batch_column_name, type_cast_function: type_cast_function) install_rename_triggers(table, old, new) end @@ -536,10 +540,10 @@ module Gitlab # table - The table containing the column. # column - The name of the column to change. # new_type - The new column type. - def change_column_type_concurrently(table, column, new_type) + def change_column_type_concurrently(table, column, new_type, type_cast_function: nil) temp_column = "#{column}_for_type_change" - rename_column_concurrently(table, column, temp_column, type: new_type) + rename_column_concurrently(table, column, temp_column, type: new_type, type_cast_function: type_cast_function) end # Performs cleanup of a concurrent type change. @@ -786,10 +790,6 @@ module Gitlab end end - def perform_background_migration_inline? - Rails.env.test? || Rails.env.development? - end - # Performs a concurrent column rename when using PostgreSQL. def install_rename_triggers_for_postgresql(trigger, table, old, new) execute <<-EOF.strip_heredoc @@ -973,106 +973,6 @@ into similar problems in the future (e.g. when new tables are created). end end - # Bulk queues background migration jobs for an entire table, batched by ID range. - # "Bulk" meaning many jobs will be pushed at a time for efficiency. - # If you need a delay interval per job, then use `queue_background_migration_jobs_by_range_at_intervals`. - # - # model_class - The table being iterated over - # job_class_name - The background migration job class as a string - # batch_size - The maximum number of rows per job - # - # Example: - # - # class Route < ActiveRecord::Base - # include EachBatch - # self.table_name = 'routes' - # end - # - # bulk_queue_background_migration_jobs_by_range(Route, 'ProcessRoutes') - # - # Where the model_class includes EachBatch, and the background migration exists: - # - # class Gitlab::BackgroundMigration::ProcessRoutes - # def perform(start_id, end_id) - # # do something - # end - # end - def bulk_queue_background_migration_jobs_by_range(model_class, job_class_name, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE) - raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') - - jobs = [] - table_name = model_class.quoted_table_name - - model_class.each_batch(of: batch_size) do |relation| - start_id, end_id = relation.pluck("MIN(#{table_name}.id)", "MAX(#{table_name}.id)").first - - if jobs.length >= BACKGROUND_MIGRATION_JOB_BUFFER_SIZE - # Note: This code path generally only helps with many millions of rows - # We push multiple jobs at a time to reduce the time spent in - # Sidekiq/Redis operations. We're using this buffer based approach so we - # don't need to run additional queries for every range. - bulk_migrate_async(jobs) - jobs.clear - end - - jobs << [job_class_name, [start_id, end_id]] - end - - bulk_migrate_async(jobs) unless jobs.empty? - end - - # Queues background migration jobs for an entire table, batched by ID range. - # Each job is scheduled with a `delay_interval` in between. - # If you use a small interval, then some jobs may run at the same time. - # - # model_class - The table or relation being iterated over - # job_class_name - The background migration job class as a string - # delay_interval - The duration between each job's scheduled time (must respond to `to_f`) - # batch_size - The maximum number of rows per job - # other_arguments - Other arguments to send to the job - # - # *Returns the final migration delay* - # - # Example: - # - # class Route < ActiveRecord::Base - # include EachBatch - # self.table_name = 'routes' - # end - # - # queue_background_migration_jobs_by_range_at_intervals(Route, 'ProcessRoutes', 1.minute) - # - # Where the model_class includes EachBatch, and the background migration exists: - # - # class Gitlab::BackgroundMigration::ProcessRoutes - # def perform(start_id, end_id) - # # do something - # end - # end - def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE, other_job_arguments: [], initial_delay: 0) - raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') - - # To not overload the worker too much we enforce a minimum interval both - # when scheduling and performing jobs. - if delay_interval < BackgroundMigrationWorker.minimum_interval - delay_interval = BackgroundMigrationWorker.minimum_interval - end - - final_delay = 0 - - model_class.each_batch(of: batch_size) do |relation, index| - start_id, end_id = relation.pluck(Arel.sql('MIN(id), MAX(id)')).first - - # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for - # the same time, which is not helpful in most cases where we wish to - # spread the work over time. - final_delay = initial_delay + delay_interval * index - migrate_in(final_delay, job_class_name, [start_id, end_id] + other_job_arguments) - end - - final_delay - end - # Fetches indexes on a column by name for postgres. # # This will include indexes using an expression on the column, for example: @@ -1131,30 +1031,6 @@ into similar problems in the future (e.g. when new tables are created). execute(sql) end - def migrate_async(*args) - with_migration_context do - BackgroundMigrationWorker.perform_async(*args) - end - end - - def migrate_in(*args) - with_migration_context do - BackgroundMigrationWorker.perform_in(*args) - end - end - - def bulk_migrate_in(*args) - with_migration_context do - BackgroundMigrationWorker.bulk_perform_in(*args) - end - end - - def bulk_migrate_async(*args) - with_migration_context do - BackgroundMigrationWorker.bulk_perform_async(*args) - end - end - # Returns the name for a check constraint # # type: @@ -1396,7 +1272,7 @@ into similar problems in the future (e.g. when new tables are created). "ON DELETE #{on_delete.upcase}" end - def create_column_from(table, old, new, type: nil, batch_column_name: :id) + def create_column_from(table, old, new, type: nil, batch_column_name: :id, type_cast_function: nil) old_col = column_for(table, old) new_type = type || old_col.type @@ -1410,7 +1286,13 @@ into similar problems in the future (e.g. when new tables are created). # necessary since we copy over old values further down. change_column_default(table, new, old_col.default) unless old_col.default.nil? - update_column_in_batches(table, new, Arel::Table.new(table)[old], batch_column_name: batch_column_name) + old_value = Arel::Table.new(table)[old] + + if type_cast_function.present? + old_value = Arel::Nodes::NamedFunction.new(type_cast_function, [old_value]) + end + + update_column_in_batches(table, new, old_value, batch_column_name: batch_column_name) add_not_null_constraint(table, new) unless old_col.null @@ -1437,10 +1319,6 @@ into similar problems in the future (e.g. when new tables are created). your migration class ERROR end - - def with_migration_context(&block) - Gitlab::ApplicationContext.with_context(caller_id: self.class.to_s, &block) - end end end end |