From 27f3465d8a52afce630417b6e4b58992b6e403fe Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 24 Mar 2021 00:09:26 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- lib/api/helpers/packages/conan/api_helpers.rb | 3 +- .../isolated_mentioned_project_parser.rb | 25 +++++ .../isolated_mentioned_user_parser.rb | 25 +++++ .../lib/gitlab/isolated_reference_extractor.rb | 2 +- .../lib/gitlab/isolated_visibility_level.rb | 60 +++++++++++ .../user_mentions/models/commit_user_mention.rb | 1 + .../models/concerns/isolated_feature_gate.rb | 20 ++++ .../models/concerns/isolated_mentionable.rb | 4 +- .../concerns/namespace/recursive_traversal.rb | 2 +- .../models/design_management/design.rb | 3 + .../user_mentions/models/design_user_mention.rb | 1 + .../user_mentions/models/epic.rb | 6 +- .../user_mentions/models/epic_user_mention.rb | 1 + .../user_mentions/models/group.rb | 2 + .../user_mentions/models/merge_request.rb | 7 +- .../models/merge_request_user_mention.rb | 1 + .../user_mentions/models/namespace.rb | 8 +- .../user_mentions/models/note.rb | 4 +- .../user_mentions/models/project.rb | 48 +++++++++ .../user_mentions/models/user.rb | 37 +++++++ .../batched_migration_runner.rb | 70 +++++++++++++ .../database/background_migration/scheduler.rb | 60 ----------- lib/gitlab/database/migration_helpers.rb | 115 +++++++++++++-------- 23 files changed, 384 insertions(+), 121 deletions(-) create mode 100644 lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb create mode 100644 lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb create mode 100644 lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb create mode 100644 lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb create mode 100644 lib/gitlab/background_migration/user_mentions/models/project.rb create mode 100644 lib/gitlab/background_migration/user_mentions/models/user.rb create mode 100644 lib/gitlab/database/background_migration/batched_migration_runner.rb delete mode 100644 lib/gitlab/database/background_migration/scheduler.rb (limited to 'lib') diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb index d5f5448fd42..24ebeb007d3 100644 --- a/lib/api/helpers/packages/conan/api_helpers.rb +++ b/lib/api/helpers/packages/conan/api_helpers.rb @@ -14,7 +14,8 @@ module API package, current_user, project, - conan_package_reference: params[:conan_package_reference] + conan_package_reference: params[:conan_package_reference], + id: params[:id] ) render_api_error!("No recipe manifest found", 404) if yield(presenter).empty? diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb new file mode 100644 index 00000000000..5930d65bc2c --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Banzai + module ReferenceParser + # isolated Banzai::ReferenceParser::MentionedGroupParser + class IsolatedMentionedProjectParser < ::Banzai::ReferenceParser::MentionedProjectParser + extend ::Gitlab::Utils::Override + + self.reference_type = :user + + override :references_relation + def references_relation + ::Gitlab::BackgroundMigration::UserMentions::Models::Project + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb new file mode 100644 index 00000000000..f5f98517433 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Banzai + module ReferenceParser + # isolated Banzai::ReferenceParser::MentionedGroupParser + class IsolatedMentionedUserParser < ::Banzai::ReferenceParser::MentionedUserParser + extend ::Gitlab::Utils::Override + + self.reference_type = :user + + override :references_relation + def references_relation + ::Gitlab::BackgroundMigration::UserMentions::Models::User + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb index 1d3a3af81a1..8610129533d 100644 --- a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb +++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb @@ -7,7 +7,7 @@ module Gitlab module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class IsolatedReferenceExtractor < ::Gitlab::ReferenceExtractor - REFERABLES = %i(isolated_mentioned_group).freeze + REFERABLES = %i(isolated_mentioned_group isolated_mentioned_user isolated_mentioned_project).freeze REFERABLES.each do |type| define_method("#{type}s") do diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb new file mode 100644 index 00000000000..0334ea1dd08 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Gitlab + # Gitlab::IsolatedVisibilityLevel module + # + # Define allowed public modes that can be used for + # GitLab projects to determine project public mode + # + module IsolatedVisibilityLevel + extend ::ActiveSupport::Concern + + included do + scope :public_to_user, -> (user = nil) do + where(visibility_level: IsolatedVisibilityLevel.levels_for_user(user)) + end + end + + PRIVATE = 0 unless const_defined?(:PRIVATE) + INTERNAL = 10 unless const_defined?(:INTERNAL) + PUBLIC = 20 unless const_defined?(:PUBLIC) + + class << self + def levels_for_user(user = nil) + return [PUBLIC] unless user + + if user.can_read_all_resources? + [PRIVATE, INTERNAL, PUBLIC] + elsif user.external? + [PUBLIC] + else + [INTERNAL, PUBLIC] + end + end + end + + def private? + visibility_level_value == PRIVATE + end + + def internal? + visibility_level_value == INTERNAL + end + + def public? + visibility_level_value == PUBLIC + end + + def visibility_level_value + self[visibility_level_field] + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb index bdb4d6c7d48..f4cc96c8bc0 100644 --- a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class CommitUserMention < ActiveRecord::Base self.table_name = 'commit_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :commit_id diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb new file mode 100644 index 00000000000..ba6b783f9f1 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + module Concerns + # isolated FeatureGate module + module IsolatedFeatureGate + def flipper_id + return if new_record? + + "#{self.class.name}:#{id}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb index be9c0ad2b3a..f684f789ea9 100644 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb @@ -70,8 +70,8 @@ module Gitlab def build_mention_values(resource_foreign_key) refs = all_references(author) - mentioned_users_ids = array_to_sql(refs.mentioned_users.pluck(:id)) - mentioned_projects_ids = array_to_sql(refs.mentioned_projects.pluck(:id)) + mentioned_users_ids = array_to_sql(refs.isolated_mentioned_users.pluck(:id)) + mentioned_projects_ids = array_to_sql(refs.isolated_mentioned_projects.pluck(:id)) mentioned_groups_ids = array_to_sql(refs.isolated_mentioned_groups.pluck(:id)) return if mentioned_users_ids.blank? && mentioned_projects_ids.blank? && mentioned_groups_ids.blank? diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb index 5cadfa45b5b..75759ed0111 100644 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb @@ -6,7 +6,7 @@ module Gitlab module Models module Concerns module Namespace - # extracted methods for recursive traversing of namespace hierarchy + # isolate recursive traversal code for namespace hierarchy module RecursiveTraversal extend ActiveSupport::Concern diff --git a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb index bdb90b5d2b9..d010d68600d 100644 --- a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb +++ b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb @@ -10,6 +10,9 @@ module Gitlab include EachBatch include Concerns::MentionableMigrationMethods + self.table_name = 'design_management_designs' + self.inheritance_column = :_type_disabled + def self.user_mention_model Gitlab::BackgroundMigration::UserMentions::Models::DesignUserMention end diff --git a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb index 68205ecd3c2..eb00f6cfa3f 100644 --- a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class DesignUserMention < ActiveRecord::Base self.table_name = 'design_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :design_id diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb index 61d9244a4c9..cfd9a4faa9b 100644 --- a/lib/gitlab/background_migration/user_mentions/models/epic.rb +++ b/lib/gitlab/background_migration/user_mentions/models/epic.rb @@ -17,10 +17,10 @@ module Gitlab cache_markdown_field :description, issuable_state_filter_enabled: true self.table_name = 'epics' + self.inheritance_column = :_type_disabled - belongs_to :author, class_name: "User" - belongs_to :project - belongs_to :group + belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" + belongs_to :group, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group" def self.user_mention_model Gitlab::BackgroundMigration::UserMentions::Models::EpicUserMention diff --git a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb index 4e3ce9bf3a7..579e4d99612 100644 --- a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class EpicUserMention < ActiveRecord::Base self.table_name = 'epic_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :epic_id diff --git a/lib/gitlab/background_migration/user_mentions/models/group.rb b/lib/gitlab/background_migration/user_mentions/models/group.rb index bc04172b9a2..a8b4b59b06c 100644 --- a/lib/gitlab/background_migration/user_mentions/models/group.rb +++ b/lib/gitlab/background_migration/user_mentions/models/group.rb @@ -7,6 +7,8 @@ module Gitlab # isolated Group model class Group < ::Gitlab::BackgroundMigration::UserMentions::Models::Namespace self.store_full_sti_class = false + self.inheritance_column = :_type_disabled + has_one :saml_provider def self.declarative_policy_class diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb index 6b52afea17c..13addcc3c55 100644 --- a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb +++ b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb @@ -17,10 +17,11 @@ module Gitlab cache_markdown_field :description, issuable_state_filter_enabled: true self.table_name = 'merge_requests' + self.inheritance_column = :_type_disabled - belongs_to :author, class_name: "User" - belongs_to :target_project, class_name: "Project" - belongs_to :source_project, class_name: "Project" + belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" + belongs_to :target_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" + belongs_to :source_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" alias_attribute :project, :target_project diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb index e9b85e9cb8c..4a85892d7b8 100644 --- a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class MergeRequestUserMention < ActiveRecord::Base self.table_name = 'merge_request_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :merge_request_id diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb index 8fa0db5fd4b..6587417d048 100644 --- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb +++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb @@ -5,9 +5,11 @@ module Gitlab module UserMentions module Models # isolated Namespace model - class Namespace < ApplicationRecord - include FeatureGate - include ::Gitlab::VisibilityLevel + class Namespace < ActiveRecord::Base + self.inheritance_column = :_type_disabled + + include Concerns::IsolatedFeatureGate + include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel include ::Gitlab::Utils::StrongMemoize include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb index a3224c8c456..7da933c7b11 100644 --- a/lib/gitlab/background_migration/user_mentions/models/note.rb +++ b/lib/gitlab/background_migration/user_mentions/models/note.rb @@ -16,9 +16,9 @@ module Gitlab attr_mentionable :note, pipeline: :note cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true - belongs_to :author, class_name: "User" + belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" belongs_to :noteable, polymorphic: true - belongs_to :project + belongs_to :project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" def for_personal_snippet? noteable && noteable.class.name == 'PersonalSnippet' diff --git a/lib/gitlab/background_migration/user_mentions/models/project.rb b/lib/gitlab/background_migration/user_mentions/models/project.rb new file mode 100644 index 00000000000..4e02bf97d12 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/project.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + # isolated Namespace model + class Project < ActiveRecord::Base + include Concerns::IsolatedFeatureGate + include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel + + self.table_name = 'projects' + self.inheritance_column = :_type_disabled + + belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id', class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group" + belongs_to :namespace, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace" + alias_method :parent, :namespace + + # Returns a collection of projects that is either public or visible to the + # logged in user. + def self.public_or_visible_to_user(user = nil, min_access_level = nil) + min_access_level = nil if user&.can_read_all_resources? + + return public_to_user unless user + + if user.is_a?(::Gitlab::BackgroundMigration::UserMentions::Models::User) + where('EXISTS (?) OR projects.visibility_level IN (?)', + user.authorizations_for_projects(min_access_level: min_access_level), + levels_for_user(user)) + end + end + + def grafana_integration + nil + end + + def default_issues_tracker? + true # we do not care of the issue tracker type(internal or external) when parsing mentions + end + + def visibility_level_field + :visibility_level + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/user.rb b/lib/gitlab/background_migration/user_mentions/models/user.rb new file mode 100644 index 00000000000..a30220b6934 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/user.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + # isolated Namespace model + class User < ActiveRecord::Base + include Concerns::IsolatedFeatureGate + + self.table_name = 'users' + self.inheritance_column = :_type_disabled + + has_many :project_authorizations, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + + def authorizations_for_projects(min_access_level: nil, related_project_column: 'projects.id') + authorizations = project_authorizations + .select(1) + .where("project_authorizations.project_id = #{related_project_column}") + + return authorizations unless min_access_level.present? + + authorizations.where('project_authorizations.access_level >= ?', min_access_level) + end + + def can_read_all_resources? + can?(:read_all_resources) + end + + def can?(action, subject = :global) + Ability.allowed?(self, action, subject) + end + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb new file mode 100644 index 00000000000..d1f208617b7 --- /dev/null +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class BatchedMigrationRunner + def initialize(migration_wrapper = BatchedMigrationWrapper.new) + @migration_wrapper = migration_wrapper + end + + def run_migration_job(active_migration) + if next_batched_job = create_next_batched_job!(active_migration) + migration_wrapper.perform(next_batched_job) + else + finish_active_migration(active_migration) + end + end + + def run_entire_migration(migration) + while migration.active? + run_migration_job(migration) + + migration.reload_last_job + end + end + + private + + attr_reader :migration_wrapper + + def create_next_batched_job!(active_migration) + next_batch_range = find_next_batch_range(active_migration) + + return if next_batch_range.nil? + + active_migration.create_batched_job!(next_batch_range.min, next_batch_range.max) + end + + def find_next_batch_range(active_migration) + batching_strategy = active_migration.batch_class.new + batch_min_value = active_migration.next_min_value + + next_batch_bounds = batching_strategy.next_batch( + active_migration.table_name, + active_migration.column_name, + batch_min_value: batch_min_value, + batch_size: active_migration.batch_size) + + return if next_batch_bounds.nil? + + clamped_batch_range(active_migration, next_batch_bounds) + end + + def clamped_batch_range(active_migration, next_bounds) + min_value, max_value = next_bounds + + return if min_value > active_migration.max_value + + max_value = max_value.clamp(min_value, active_migration.max_value) + + (min_value..max_value) + end + + def finish_active_migration(active_migration) + active_migration.finished! + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/scheduler.rb b/lib/gitlab/database/background_migration/scheduler.rb deleted file mode 100644 index 5f8a5ec06a5..00000000000 --- a/lib/gitlab/database/background_migration/scheduler.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module BackgroundMigration - class Scheduler - def perform(migration_wrapper: BatchedMigrationWrapper.new) - active_migration = BatchedMigration.active.queue_order.first - - return unless active_migration&.interval_elapsed? - - if next_batched_job = create_next_batched_job!(active_migration) - migration_wrapper.perform(next_batched_job) - else - finish_active_migration(active_migration) - end - end - - private - - def create_next_batched_job!(active_migration) - next_batch_range = find_next_batch_range(active_migration) - - return if next_batch_range.nil? - - active_migration.create_batched_job!(next_batch_range.min, next_batch_range.max) - end - - def find_next_batch_range(active_migration) - batching_strategy = active_migration.batch_class.new - batch_min_value = active_migration.next_min_value - - next_batch_bounds = batching_strategy.next_batch( - active_migration.table_name, - active_migration.column_name, - batch_min_value: batch_min_value, - batch_size: active_migration.batch_size) - - return if next_batch_bounds.nil? - - clamped_batch_range(active_migration, next_batch_bounds) - end - - def clamped_batch_range(active_migration, next_bounds) - min_value, max_value = next_bounds - - return if min_value > active_migration.max_value - - max_value = max_value.clamp(min_value, active_migration.max_value) - - (min_value..max_value) - end - - def finish_active_migration(active_migration) - active_migration.finished! - end - end - end - end -end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 31e733050e1..bc50f0c3c04 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -4,6 +4,7 @@ module Gitlab module Database module MigrationHelpers include Migrations::BackgroundMigrationHelpers + include DynamicModelHelpers # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS MAX_IDENTIFIER_NAME_LENGTH = 63 @@ -927,19 +928,67 @@ module Gitlab # This is crucial for Primary Key conversions, because setting a column # as the PK converts even check constraints to NOT NULL constraints # and forces an inline re-verification of the whole table. - # - It backfills the new column with the values of the existing primary key - # by scheduling background jobs. - # - It tracks the scheduled background jobs through the use of - # Gitlab::Database::BackgroundMigrationJob + # - It sets up a trigger to keep the two columns in sync. + # + # Note: this helper is intended to be used in a regular (pre-deployment) migration. + # + # This helper is part 1 of a multi-step migration process: + # 1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers + # 2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations + # 3. remaining steps TBD, see #288005 + # + # table - The name of the database table containing the column + # column - The name of the column that we want to convert to bigint. + # primary_key - The name of the primary key column (most often :id) + def initialize_conversion_of_integer_to_bigint(table, column, primary_key: :id) + unless table_exists?(table) + raise "Table #{table} does not exist" + end + + unless column_exists?(table, primary_key) + raise "Column #{primary_key} does not exist on #{table}" + end + + unless column_exists?(table, column) + raise "Column #{column} does not exist on #{table}" + end + + check_trigger_permissions!(table) + + old_column = column_for(table, column) + tmp_column = "#{column}_convert_to_bigint" + + with_lock_retries do + if (column.to_s == primary_key.to_s) || !old_column.null + # If the column to be converted is either a PK or is defined as NOT NULL, + # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow + # That way, we skip the expensive validation step required to add + # a NOT NULL constraint at the end of the process + add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false) + else + add_column(table, tmp_column, :bigint, default: old_column.default) + end + + install_rename_triggers(table, column, tmp_column) + end + end + + # Backfills the new column used in the conversion of an integer column to bigint using background migrations. + # + # - This helper should be called from a post-deployment migration. + # - In order for this helper to work properly, the new column must be first initialized with + # the `initialize_conversion_of_integer_to_bigint` helper. + # - It tracks the scheduled background jobs through Gitlab::Database::BackgroundMigration::BatchedMigration, # which allows a more thorough check that all jobs succeeded in the # cleanup migration and is way faster for very large tables. - # - It sets up a trigger to keep the two columns in sync - # - It does not schedule a cleanup job: we have to do that with followup - # post deployment migrations in the next release. # - # This needs to be done manually by using the - # `cleanup_initialize_conversion_of_integer_to_bigint` - # (not yet implemented - check #288005) + # Note: this helper is intended to be used in a post-deployment migration, to ensure any new code is + # deployed (including background job changes) before we begin processing the background migration. + # + # This helper is part 2 of a multi-step migration process: + # 1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers + # 2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations + # 3. remaining steps TBD, see #288005 # # table - The name of the database table containing the column # column - The name of the column that we want to convert to bigint. @@ -960,7 +1009,7 @@ module Gitlab # and set the batch_size to 50_000 which will require # ~50s = (50000 / 200) * (0.1 + 0.1) to complete and leaves breathing space # between the scheduled jobs - def initialize_conversion_of_integer_to_bigint( + def backfill_conversion_of_integer_to_bigint( table, column, primary_key: :id, @@ -969,10 +1018,6 @@ module Gitlab interval: 2.minutes ) - if transaction_open? - raise 'initialize_conversion_of_integer_to_bigint can not be run inside a transaction' - end - unless table_exists?(table) raise "Table #{table} does not exist" end @@ -985,46 +1030,26 @@ module Gitlab raise "Column #{column} does not exist on #{table}" end - check_trigger_permissions!(table) - - old_column = column_for(table, column) tmp_column = "#{column}_convert_to_bigint" - with_lock_retries do - if (column.to_s == primary_key.to_s) || !old_column.null - # If the column to be converted is either a PK or is defined as NOT NULL, - # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow - # That way, we skip the expensive validation step required to add - # a NOT NULL constraint at the end of the process - add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false) - else - add_column(table, tmp_column, :bigint, default: old_column.default) - end - - install_rename_triggers(table, column, tmp_column) - end - - source_model = Class.new(ActiveRecord::Base) do - include EachBatch - - self.table_name = table - self.inheritance_column = :_type_disabled + unless column_exists?(table, tmp_column) + raise 'The temporary column does not exist, initialize it with `initialize_conversion_of_integer_to_bigint`' end - queue_background_migration_jobs_by_range_at_intervals( - source_model, + batched_migration = queue_batched_background_migration( 'CopyColumnUsingBackgroundMigrationJob', - interval, + table, + primary_key, + column, + tmp_column, + job_interval: interval, batch_size: batch_size, - other_job_arguments: [table, primary_key, sub_batch_size, column, tmp_column], - track_jobs: true, - primary_column_name: primary_key - ) + sub_batch_size: sub_batch_size) if perform_background_migration_inline? # To ensure the schema is up to date immediately we perform the # migration inline in dev / test environments. - Gitlab::BackgroundMigration.steal('CopyColumnUsingBackgroundMigrationJob') + Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new.run_entire_migration(batched_migration) end end -- cgit v1.2.1