diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-09-20 13:18:24 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-09-20 13:18:24 +0000 |
commit | 0653e08efd039a5905f3fa4f6e9cef9f5d2f799c (patch) | |
tree | 4dcc884cf6d81db44adae4aa99f8ec1233a41f55 /lib/gitlab/background_migration | |
parent | 744144d28e3e7fddc117924fef88de5d9674fe4c (diff) | |
download | gitlab-ce-14.3.0-rc42.tar.gz |
Add latest changes from gitlab-org/gitlab@14-3-stable-eev14.3.0-rc42
Diffstat (limited to 'lib/gitlab/background_migration')
8 files changed, 163 insertions, 4 deletions
diff --git a/lib/gitlab/background_migration/backfill_design_internal_ids.rb b/lib/gitlab/background_migration/backfill_design_internal_ids.rb index 6d1df95c66d..236c6b6eb9a 100644 --- a/lib/gitlab/background_migration/backfill_design_internal_ids.rb +++ b/lib/gitlab/background_migration/backfill_design_internal_ids.rb @@ -73,7 +73,7 @@ module Gitlab # violation. We can safely roll-back the nested transaction and perform # a lookup instead to retrieve the record. def create_record - subject.transaction(requires_new: true) do + subject.transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions InternalId.create!( **scope, usage: usage_value, diff --git a/lib/gitlab/background_migration/backfill_iteration_cadence_id_for_boards.rb b/lib/gitlab/background_migration/backfill_iteration_cadence_id_for_boards.rb new file mode 100644 index 00000000000..67f4690868e --- /dev/null +++ b/lib/gitlab/background_migration/backfill_iteration_cadence_id_for_boards.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class BackfillIterationCadenceIdForBoards + def perform(*args) + end + end + end +end + +Gitlab::BackgroundMigration::BackfillIterationCadenceIdForBoards.prepend_mod_with('Gitlab::BackgroundMigration::BackfillIterationCadenceIdForBoards') diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index f5c8796bd18..a9eaeb0562d 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -21,7 +21,7 @@ module Gitlab shard_id = shards.fetch(name, nil) return shard_id if shard_id.present? - Shard.transaction(requires_new: true) do + Shard.transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions create!(name) end rescue ActiveRecord::RecordNotUnique diff --git a/lib/gitlab/background_migration/backfill_projects_with_coverage.rb b/lib/gitlab/background_migration/backfill_projects_with_coverage.rb new file mode 100644 index 00000000000..ca262c0bd59 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_projects_with_coverage.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill project_ci_feature_usages for a range of projects with coverage + class BackfillProjectsWithCoverage + class ProjectCiFeatureUsage < ActiveRecord::Base # rubocop:disable Style/Documentation + self.table_name = 'project_ci_feature_usages' + end + + COVERAGE_ENUM_VALUE = 1 + INSERT_DELAY_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + report_results = ActiveRecord::Base.connection.execute <<~SQL + SELECT DISTINCT project_id, default_branch + FROM ci_daily_build_group_report_results + WHERE id BETWEEN #{start_id} AND #{end_id} + SQL + + report_results.to_a.in_groups_of(sub_batch_size, false) do |batch| + ProjectCiFeatureUsage.insert_all(build_values(batch)) + + sleep INSERT_DELAY_SECONDS + end + end + + private + + def build_values(batch) + batch.map do |data| + { + project_id: data['project_id'], + feature: COVERAGE_ENUM_VALUE, + default_branch: data['default_branch'] + } + end + end + end + end +end diff --git a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb new file mode 100644 index 00000000000..31b5b5cdb73 --- /dev/null +++ b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to extract the project topics into a separate `topics` table + class ExtractProjectTopicsIntoSeparateTable + # Temporary AR table for tags + class Tag < ActiveRecord::Base + self.table_name = 'tags' + end + + # Temporary AR table for taggings + class Tagging < ActiveRecord::Base + self.table_name = 'taggings' + belongs_to :tag + end + + # Temporary AR table for topics + class Topic < ActiveRecord::Base + self.table_name = 'topics' + end + + # Temporary AR table for project topics + class ProjectTopic < ActiveRecord::Base + self.table_name = 'project_topics' + belongs_to :topic + end + + # Temporary AR table for projects + class Project < ActiveRecord::Base + self.table_name = 'projects' + end + + def perform(start_id, stop_id) + Tagging.includes(:tag).where(taggable_type: 'Project', id: start_id..stop_id).each do |tagging| + if Project.exists?(id: tagging.taggable_id) && tagging.tag + begin + topic = Topic.find_or_create_by(name: tagging.tag.name) + project_topic = ProjectTopic.find_or_create_by(project_id: tagging.taggable_id, topic: topic) + + tagging.delete if project_topic.persisted? + rescue StandardError => e + Gitlab::ErrorTracking.log_exception(e, tagging_id: tagging.id) + end + else + tagging.delete + end + end + + mark_job_as_succeeded(start_id, stop_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb index c096dae0631..3605b157f4f 100644 --- a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb +++ b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb @@ -14,7 +14,7 @@ module Gitlab mail( template_path: 'unconfirm_mailer', template_name: 'unconfirm_notification_email', - to: @user.notification_email, + to: @user.notification_email_or_default, subject: subject('GitLab email verification request') ) end diff --git a/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb index e694e5359cd..7d150b9cd83 100644 --- a/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb +++ b/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb @@ -14,7 +14,7 @@ module Gitlab # The number of rows in merge_request_diff_commits to get in a single # query. - COMMIT_ROWS_PER_QUERY = 10_000 + COMMIT_ROWS_PER_QUERY = 1_000 # The number of rows in merge_request_diff_commits to update in a single # query. @@ -78,6 +78,8 @@ module Gitlab # rubocop: enable Style/Documentation def perform(start_id, stop_id) + return if already_processed?(start_id, stop_id) + # This Hash maps user names + emails to their corresponding rows in # merge_request_diff_commit_users. user_mapping = {} @@ -94,6 +96,13 @@ module Gitlab ) end + def already_processed?(start_id, stop_id) + Database::BackgroundMigrationJob + .for_migration_execution('MigrateMergeRequestDiffCommitUsers', [start_id, stop_id]) + .succeeded + .any? + end + # Returns the data we'll use to determine what merge_request_diff_commits # rows to update, and what data to use for populating their # commit_author_id and committer_id columns. diff --git a/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb new file mode 100644 index 00000000000..43a7032e682 --- /dev/null +++ b/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A background migration that finished any pending + # MigrateMergeRequestDiffCommitUsers jobs, and schedules new jobs itself. + # + # This migration exists so we can bypass rescheduling issues (e.g. jobs + # getting dropped after too many retries) that may occur when + # MigrateMergeRequestDiffCommitUsers jobs take longer than expected. + class StealMigrateMergeRequestDiffCommitUsers + def perform(start_id, stop_id) + MigrateMergeRequestDiffCommitUsers.new.perform(start_id, stop_id) + schedule_next_job + end + + def schedule_next_job + next_job = Database::BackgroundMigrationJob + .for_migration_class('MigrateMergeRequestDiffCommitUsers') + .pending + .first + + return unless next_job + + BackgroundMigrationWorker.perform_in( + 5.minutes, + 'StealMigrateMergeRequestDiffCommitUsers', + next_job.arguments + ) + end + end + end +end |