summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2021-03-24 00:09:26 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2021-03-24 00:09:26 +0000
commit27f3465d8a52afce630417b6e4b58992b6e403fe (patch)
tree55a48a13e9db76ca804ecba4b5ffa12a65076944 /lib
parent78f935d56652eee385683bf7a55c3b9a86a63a0e (diff)
downloadgitlab-ce-27f3465d8a52afce630417b6e4b58992b6e403fe.tar.gz
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r--lib/api/helpers/packages/conan/api_helpers.rb3
-rw-r--r--lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb25
-rw-r--r--lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb25
-rw-r--r--lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb2
-rw-r--r--lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb60
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb1
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb20
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb4
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb2
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/design_management/design.rb3
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb1
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/epic.rb6
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb1
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/group.rb2
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/merge_request.rb7
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb1
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/namespace.rb8
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/note.rb4
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/project.rb48
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/user.rb37
-rw-r--r--lib/gitlab/database/background_migration/batched_migration_runner.rb (renamed from lib/gitlab/database/background_migration/scheduler.rb)20
-rw-r--r--lib/gitlab/database/migration_helpers.rb115
22 files changed, 329 insertions, 66 deletions
diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb
index d5f5448fd42..24ebeb007d3 100644
--- a/lib/api/helpers/packages/conan/api_helpers.rb
+++ b/lib/api/helpers/packages/conan/api_helpers.rb
@@ -14,7 +14,8 @@ module API
package,
current_user,
project,
- conan_package_reference: params[:conan_package_reference]
+ conan_package_reference: params[:conan_package_reference],
+ id: params[:id]
)
render_api_error!("No recipe manifest found", 404) if yield(presenter).empty?
diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb
new file mode 100644
index 00000000000..5930d65bc2c
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Lib
+ module Banzai
+ module ReferenceParser
+ # isolated Banzai::ReferenceParser::MentionedGroupParser
+ class IsolatedMentionedProjectParser < ::Banzai::ReferenceParser::MentionedProjectParser
+ extend ::Gitlab::Utils::Override
+
+ self.reference_type = :user
+
+ override :references_relation
+ def references_relation
+ ::Gitlab::BackgroundMigration::UserMentions::Models::Project
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb
new file mode 100644
index 00000000000..f5f98517433
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Lib
+ module Banzai
+ module ReferenceParser
+ # isolated Banzai::ReferenceParser::MentionedGroupParser
+ class IsolatedMentionedUserParser < ::Banzai::ReferenceParser::MentionedUserParser
+ extend ::Gitlab::Utils::Override
+
+ self.reference_type = :user
+
+ override :references_relation
+ def references_relation
+ ::Gitlab::BackgroundMigration::UserMentions::Models::User
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb
index 1d3a3af81a1..8610129533d 100644
--- a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb
+++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb
@@ -7,7 +7,7 @@ module Gitlab
module Gitlab
# Extract possible GFM references from an arbitrary String for further processing.
class IsolatedReferenceExtractor < ::Gitlab::ReferenceExtractor
- REFERABLES = %i(isolated_mentioned_group).freeze
+ REFERABLES = %i(isolated_mentioned_group isolated_mentioned_user isolated_mentioned_project).freeze
REFERABLES.each do |type|
define_method("#{type}s") do
diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb
new file mode 100644
index 00000000000..0334ea1dd08
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Lib
+ module Gitlab
+ # Gitlab::IsolatedVisibilityLevel module
+ #
+ # Define allowed public modes that can be used for
+ # GitLab projects to determine project public mode
+ #
+ module IsolatedVisibilityLevel
+ extend ::ActiveSupport::Concern
+
+ included do
+ scope :public_to_user, -> (user = nil) do
+ where(visibility_level: IsolatedVisibilityLevel.levels_for_user(user))
+ end
+ end
+
+ PRIVATE = 0 unless const_defined?(:PRIVATE)
+ INTERNAL = 10 unless const_defined?(:INTERNAL)
+ PUBLIC = 20 unless const_defined?(:PUBLIC)
+
+ class << self
+ def levels_for_user(user = nil)
+ return [PUBLIC] unless user
+
+ if user.can_read_all_resources?
+ [PRIVATE, INTERNAL, PUBLIC]
+ elsif user.external?
+ [PUBLIC]
+ else
+ [INTERNAL, PUBLIC]
+ end
+ end
+ end
+
+ def private?
+ visibility_level_value == PRIVATE
+ end
+
+ def internal?
+ visibility_level_value == INTERNAL
+ end
+
+ def public?
+ visibility_level_value == PUBLIC
+ end
+
+ def visibility_level_value
+ self[visibility_level_field]
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb
index bdb4d6c7d48..f4cc96c8bc0 100644
--- a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb
@@ -7,6 +7,7 @@ module Gitlab
module Models
class CommitUserMention < ActiveRecord::Base
self.table_name = 'commit_user_mentions'
+ self.inheritance_column = :_type_disabled
def self.resource_foreign_key
:commit_id
diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb
new file mode 100644
index 00000000000..ba6b783f9f1
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Models
+ module Concerns
+ # isolated FeatureGate module
+ module IsolatedFeatureGate
+ def flipper_id
+ return if new_record?
+
+ "#{self.class.name}:#{id}"
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb
index be9c0ad2b3a..f684f789ea9 100644
--- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb
@@ -70,8 +70,8 @@ module Gitlab
def build_mention_values(resource_foreign_key)
refs = all_references(author)
- mentioned_users_ids = array_to_sql(refs.mentioned_users.pluck(:id))
- mentioned_projects_ids = array_to_sql(refs.mentioned_projects.pluck(:id))
+ mentioned_users_ids = array_to_sql(refs.isolated_mentioned_users.pluck(:id))
+ mentioned_projects_ids = array_to_sql(refs.isolated_mentioned_projects.pluck(:id))
mentioned_groups_ids = array_to_sql(refs.isolated_mentioned_groups.pluck(:id))
return if mentioned_users_ids.blank? && mentioned_projects_ids.blank? && mentioned_groups_ids.blank?
diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb
index 5cadfa45b5b..75759ed0111 100644
--- a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb
@@ -6,7 +6,7 @@ module Gitlab
module Models
module Concerns
module Namespace
- # extracted methods for recursive traversing of namespace hierarchy
+ # isolate recursive traversal code for namespace hierarchy
module RecursiveTraversal
extend ActiveSupport::Concern
diff --git a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb
index bdb90b5d2b9..d010d68600d 100644
--- a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb
@@ -10,6 +10,9 @@ module Gitlab
include EachBatch
include Concerns::MentionableMigrationMethods
+ self.table_name = 'design_management_designs'
+ self.inheritance_column = :_type_disabled
+
def self.user_mention_model
Gitlab::BackgroundMigration::UserMentions::Models::DesignUserMention
end
diff --git a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb
index 68205ecd3c2..eb00f6cfa3f 100644
--- a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb
@@ -7,6 +7,7 @@ module Gitlab
module Models
class DesignUserMention < ActiveRecord::Base
self.table_name = 'design_user_mentions'
+ self.inheritance_column = :_type_disabled
def self.resource_foreign_key
:design_id
diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb
index 61d9244a4c9..cfd9a4faa9b 100644
--- a/lib/gitlab/background_migration/user_mentions/models/epic.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/epic.rb
@@ -17,10 +17,10 @@ module Gitlab
cache_markdown_field :description, issuable_state_filter_enabled: true
self.table_name = 'epics'
+ self.inheritance_column = :_type_disabled
- belongs_to :author, class_name: "User"
- belongs_to :project
- belongs_to :group
+ belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User"
+ belongs_to :group, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group"
def self.user_mention_model
Gitlab::BackgroundMigration::UserMentions::Models::EpicUserMention
diff --git a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb
index 4e3ce9bf3a7..579e4d99612 100644
--- a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb
@@ -7,6 +7,7 @@ module Gitlab
module Models
class EpicUserMention < ActiveRecord::Base
self.table_name = 'epic_user_mentions'
+ self.inheritance_column = :_type_disabled
def self.resource_foreign_key
:epic_id
diff --git a/lib/gitlab/background_migration/user_mentions/models/group.rb b/lib/gitlab/background_migration/user_mentions/models/group.rb
index bc04172b9a2..a8b4b59b06c 100644
--- a/lib/gitlab/background_migration/user_mentions/models/group.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/group.rb
@@ -7,6 +7,8 @@ module Gitlab
# isolated Group model
class Group < ::Gitlab::BackgroundMigration::UserMentions::Models::Namespace
self.store_full_sti_class = false
+ self.inheritance_column = :_type_disabled
+
has_one :saml_provider
def self.declarative_policy_class
diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb
index 6b52afea17c..13addcc3c55 100644
--- a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb
@@ -17,10 +17,11 @@ module Gitlab
cache_markdown_field :description, issuable_state_filter_enabled: true
self.table_name = 'merge_requests'
+ self.inheritance_column = :_type_disabled
- belongs_to :author, class_name: "User"
- belongs_to :target_project, class_name: "Project"
- belongs_to :source_project, class_name: "Project"
+ belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User"
+ belongs_to :target_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project"
+ belongs_to :source_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project"
alias_attribute :project, :target_project
diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb
index e9b85e9cb8c..4a85892d7b8 100644
--- a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb
@@ -7,6 +7,7 @@ module Gitlab
module Models
class MergeRequestUserMention < ActiveRecord::Base
self.table_name = 'merge_request_user_mentions'
+ self.inheritance_column = :_type_disabled
def self.resource_foreign_key
:merge_request_id
diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb
index 8fa0db5fd4b..6587417d048 100644
--- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb
@@ -5,9 +5,11 @@ module Gitlab
module UserMentions
module Models
# isolated Namespace model
- class Namespace < ApplicationRecord
- include FeatureGate
- include ::Gitlab::VisibilityLevel
+ class Namespace < ActiveRecord::Base
+ self.inheritance_column = :_type_disabled
+
+ include Concerns::IsolatedFeatureGate
+ include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel
include ::Gitlab::Utils::StrongMemoize
include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal
diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb
index a3224c8c456..7da933c7b11 100644
--- a/lib/gitlab/background_migration/user_mentions/models/note.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/note.rb
@@ -16,9 +16,9 @@ module Gitlab
attr_mentionable :note, pipeline: :note
cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true
- belongs_to :author, class_name: "User"
+ belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User"
belongs_to :noteable, polymorphic: true
- belongs_to :project
+ belongs_to :project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project"
def for_personal_snippet?
noteable && noteable.class.name == 'PersonalSnippet'
diff --git a/lib/gitlab/background_migration/user_mentions/models/project.rb b/lib/gitlab/background_migration/user_mentions/models/project.rb
new file mode 100644
index 00000000000..4e02bf97d12
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/project.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Models
+ # isolated Namespace model
+ class Project < ActiveRecord::Base
+ include Concerns::IsolatedFeatureGate
+ include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel
+
+ self.table_name = 'projects'
+ self.inheritance_column = :_type_disabled
+
+ belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id', class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group"
+ belongs_to :namespace, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace"
+ alias_method :parent, :namespace
+
+ # Returns a collection of projects that is either public or visible to the
+ # logged in user.
+ def self.public_or_visible_to_user(user = nil, min_access_level = nil)
+ min_access_level = nil if user&.can_read_all_resources?
+
+ return public_to_user unless user
+
+ if user.is_a?(::Gitlab::BackgroundMigration::UserMentions::Models::User)
+ where('EXISTS (?) OR projects.visibility_level IN (?)',
+ user.authorizations_for_projects(min_access_level: min_access_level),
+ levels_for_user(user))
+ end
+ end
+
+ def grafana_integration
+ nil
+ end
+
+ def default_issues_tracker?
+ true # we do not care of the issue tracker type(internal or external) when parsing mentions
+ end
+
+ def visibility_level_field
+ :visibility_level
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/user.rb b/lib/gitlab/background_migration/user_mentions/models/user.rb
new file mode 100644
index 00000000000..a30220b6934
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/user.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Models
+ # isolated Namespace model
+ class User < ActiveRecord::Base
+ include Concerns::IsolatedFeatureGate
+
+ self.table_name = 'users'
+ self.inheritance_column = :_type_disabled
+
+ has_many :project_authorizations, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent
+
+ def authorizations_for_projects(min_access_level: nil, related_project_column: 'projects.id')
+ authorizations = project_authorizations
+ .select(1)
+ .where("project_authorizations.project_id = #{related_project_column}")
+
+ return authorizations unless min_access_level.present?
+
+ authorizations.where('project_authorizations.access_level >= ?', min_access_level)
+ end
+
+ def can_read_all_resources?
+ can?(:read_all_resources)
+ end
+
+ def can?(action, subject = :global)
+ Ability.allowed?(self, action, subject)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/background_migration/scheduler.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb
index 5f8a5ec06a5..d1f208617b7 100644
--- a/lib/gitlab/database/background_migration/scheduler.rb
+++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb
@@ -3,12 +3,12 @@
module Gitlab
module Database
module BackgroundMigration
- class Scheduler
- def perform(migration_wrapper: BatchedMigrationWrapper.new)
- active_migration = BatchedMigration.active.queue_order.first
-
- return unless active_migration&.interval_elapsed?
+ class BatchedMigrationRunner
+ def initialize(migration_wrapper = BatchedMigrationWrapper.new)
+ @migration_wrapper = migration_wrapper
+ end
+ def run_migration_job(active_migration)
if next_batched_job = create_next_batched_job!(active_migration)
migration_wrapper.perform(next_batched_job)
else
@@ -16,8 +16,18 @@ module Gitlab
end
end
+ def run_entire_migration(migration)
+ while migration.active?
+ run_migration_job(migration)
+
+ migration.reload_last_job
+ end
+ end
+
private
+ attr_reader :migration_wrapper
+
def create_next_batched_job!(active_migration)
next_batch_range = find_next_batch_range(active_migration)
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 31e733050e1..bc50f0c3c04 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -4,6 +4,7 @@ module Gitlab
module Database
module MigrationHelpers
include Migrations::BackgroundMigrationHelpers
+ include DynamicModelHelpers
# https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
MAX_IDENTIFIER_NAME_LENGTH = 63
@@ -927,19 +928,67 @@ module Gitlab
# This is crucial for Primary Key conversions, because setting a column
# as the PK converts even check constraints to NOT NULL constraints
# and forces an inline re-verification of the whole table.
- # - It backfills the new column with the values of the existing primary key
- # by scheduling background jobs.
- # - It tracks the scheduled background jobs through the use of
- # Gitlab::Database::BackgroundMigrationJob
+ # - It sets up a trigger to keep the two columns in sync.
+ #
+ # Note: this helper is intended to be used in a regular (pre-deployment) migration.
+ #
+ # This helper is part 1 of a multi-step migration process:
+ # 1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers
+ # 2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations
+ # 3. remaining steps TBD, see #288005
+ #
+ # table - The name of the database table containing the column
+ # column - The name of the column that we want to convert to bigint.
+ # primary_key - The name of the primary key column (most often :id)
+ def initialize_conversion_of_integer_to_bigint(table, column, primary_key: :id)
+ unless table_exists?(table)
+ raise "Table #{table} does not exist"
+ end
+
+ unless column_exists?(table, primary_key)
+ raise "Column #{primary_key} does not exist on #{table}"
+ end
+
+ unless column_exists?(table, column)
+ raise "Column #{column} does not exist on #{table}"
+ end
+
+ check_trigger_permissions!(table)
+
+ old_column = column_for(table, column)
+ tmp_column = "#{column}_convert_to_bigint"
+
+ with_lock_retries do
+ if (column.to_s == primary_key.to_s) || !old_column.null
+ # If the column to be converted is either a PK or is defined as NOT NULL,
+ # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow
+ # That way, we skip the expensive validation step required to add
+ # a NOT NULL constraint at the end of the process
+ add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false)
+ else
+ add_column(table, tmp_column, :bigint, default: old_column.default)
+ end
+
+ install_rename_triggers(table, column, tmp_column)
+ end
+ end
+
+ # Backfills the new column used in the conversion of an integer column to bigint using background migrations.
+ #
+ # - This helper should be called from a post-deployment migration.
+ # - In order for this helper to work properly, the new column must be first initialized with
+ # the `initialize_conversion_of_integer_to_bigint` helper.
+ # - It tracks the scheduled background jobs through Gitlab::Database::BackgroundMigration::BatchedMigration,
# which allows a more thorough check that all jobs succeeded in the
# cleanup migration and is way faster for very large tables.
- # - It sets up a trigger to keep the two columns in sync
- # - It does not schedule a cleanup job: we have to do that with followup
- # post deployment migrations in the next release.
#
- # This needs to be done manually by using the
- # `cleanup_initialize_conversion_of_integer_to_bigint`
- # (not yet implemented - check #288005)
+ # Note: this helper is intended to be used in a post-deployment migration, to ensure any new code is
+ # deployed (including background job changes) before we begin processing the background migration.
+ #
+ # This helper is part 2 of a multi-step migration process:
+ # 1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers
+ # 2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations
+ # 3. remaining steps TBD, see #288005
#
# table - The name of the database table containing the column
# column - The name of the column that we want to convert to bigint.
@@ -960,7 +1009,7 @@ module Gitlab
# and set the batch_size to 50_000 which will require
# ~50s = (50000 / 200) * (0.1 + 0.1) to complete and leaves breathing space
# between the scheduled jobs
- def initialize_conversion_of_integer_to_bigint(
+ def backfill_conversion_of_integer_to_bigint(
table,
column,
primary_key: :id,
@@ -969,10 +1018,6 @@ module Gitlab
interval: 2.minutes
)
- if transaction_open?
- raise 'initialize_conversion_of_integer_to_bigint can not be run inside a transaction'
- end
-
unless table_exists?(table)
raise "Table #{table} does not exist"
end
@@ -985,46 +1030,26 @@ module Gitlab
raise "Column #{column} does not exist on #{table}"
end
- check_trigger_permissions!(table)
-
- old_column = column_for(table, column)
tmp_column = "#{column}_convert_to_bigint"
- with_lock_retries do
- if (column.to_s == primary_key.to_s) || !old_column.null
- # If the column to be converted is either a PK or is defined as NOT NULL,
- # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow
- # That way, we skip the expensive validation step required to add
- # a NOT NULL constraint at the end of the process
- add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false)
- else
- add_column(table, tmp_column, :bigint, default: old_column.default)
- end
-
- install_rename_triggers(table, column, tmp_column)
- end
-
- source_model = Class.new(ActiveRecord::Base) do
- include EachBatch
-
- self.table_name = table
- self.inheritance_column = :_type_disabled
+ unless column_exists?(table, tmp_column)
+ raise 'The temporary column does not exist, initialize it with `initialize_conversion_of_integer_to_bigint`'
end
- queue_background_migration_jobs_by_range_at_intervals(
- source_model,
+ batched_migration = queue_batched_background_migration(
'CopyColumnUsingBackgroundMigrationJob',
- interval,
+ table,
+ primary_key,
+ column,
+ tmp_column,
+ job_interval: interval,
batch_size: batch_size,
- other_job_arguments: [table, primary_key, sub_batch_size, column, tmp_column],
- track_jobs: true,
- primary_column_name: primary_key
- )
+ sub_batch_size: sub_batch_size)
if perform_background_migration_inline?
# To ensure the schema is up to date immediately we perform the
# migration inline in dev / test environments.
- Gitlab::BackgroundMigration.steal('CopyColumnUsingBackgroundMigrationJob')
+ Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new.run_entire_migration(batched_migration)
end
end