diff options
7 files changed, 199 insertions, 0 deletions
diff --git a/changelogs/unreleased/feature-gb-pipeline-variable-expressions.yml b/changelogs/unreleased/feature-gb-pipeline-variable-expressions.yml new file mode 100644 index 00000000000..28820649af3 --- /dev/null +++ b/changelogs/unreleased/feature-gb-pipeline-variable-expressions.yml @@ -0,0 +1,5 @@ +--- +title: Add catch-up background migration to migrate pipeline stages +merge_request: 15741 +author: +type: performance diff --git a/db/post_migrate/20180212101828_add_tmp_partial_null_index_to_builds.rb b/db/post_migrate/20180212101828_add_tmp_partial_null_index_to_builds.rb new file mode 100644 index 00000000000..e55e2e6f888 --- /dev/null +++ b/db/post_migrate/20180212101828_add_tmp_partial_null_index_to_builds.rb @@ -0,0 +1,14 @@ +class AddTmpPartialNullIndexToBuilds < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + add_concurrent_index(:ci_builds, :id, where: 'stage_id IS NULL', + name: 'tmp_id_partial_null_index') + end + + def down + remove_concurrent_index_by_name(:ci_builds, 'tmp_id_partial_null_index') + end +end diff --git a/db/post_migrate/20180212101928_schedule_build_stage_migration.rb b/db/post_migrate/20180212101928_schedule_build_stage_migration.rb new file mode 100644 index 00000000000..df15b2cd9d4 --- /dev/null +++ b/db/post_migrate/20180212101928_schedule_build_stage_migration.rb @@ -0,0 +1,29 @@ +class ScheduleBuildStageMigration < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + MIGRATION = 'MigrateBuildStage'.freeze + BATCH_SIZE = 500 + + disable_ddl_transaction! + + class Build < ActiveRecord::Base + include EachBatch + self.table_name = 'ci_builds' + end + + def up + disable_statement_timeout + + Build.where('stage_id IS NULL').tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + MIGRATION, + 5.minutes, + batch_size: BATCH_SIZE) + end + end + + def down + # noop + end +end diff --git a/db/post_migrate/20180212102028_remove_tmp_partial_null_index_from_builds.rb b/db/post_migrate/20180212102028_remove_tmp_partial_null_index_from_builds.rb new file mode 100644 index 00000000000..ed7b1fc72f4 --- /dev/null +++ b/db/post_migrate/20180212102028_remove_tmp_partial_null_index_from_builds.rb @@ -0,0 +1,14 @@ +class RemoveTmpPartialNullIndexFromBuilds < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name(:ci_builds, 'tmp_id_partial_null_index') + end + + def down + add_concurrent_index(:ci_builds, :id, where: 'stage_id IS NULL', + name: 'tmp_id_partial_null_index') + end +end diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb new file mode 100644 index 00000000000..8fe4f1a2289 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_build_stage.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class MigrateBuildStage + module Migratable + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' + end + + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + + def ensure_stage!(attempts: 2) + find_stage || create_stage! + rescue ActiveRecord::RecordNotUnique + retry if (attempts -= 1) > 0 + raise + end + + def find_stage + Stage.find_by(name: self.stage || 'test', + pipeline_id: self.commit_id, + project_id: self.project_id) + end + + def create_stage! + Stage.create!(name: self.stage || 'test', + pipeline_id: self.commit_id, + project_id: self.project_id) + end + end + end + + def perform(start_id, stop_id) + stages = Migratable::Build.where('stage_id IS NULL') + .where('id BETWEEN ? AND ?', start_id, stop_id) + .map { |build| build.ensure_stage! } + .compact.map(&:id) + + MigrateBuildStageIdReference.new.perform(start_id, stop_id) + MigrateStageStatus.new.perform(stages.min, stages.max) + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb b/spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb new file mode 100644 index 00000000000..e112e9e9e3d --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb @@ -0,0 +1,54 @@ +require 'spec_helper' + +describe Gitlab::BackgroundMigration::MigrateBuildStage, :migration, schema: 20180212101928 do + let(:projects) { table(:projects) } + let(:pipelines) { table(:ci_pipelines) } + let(:stages) { table(:ci_stages) } + let(:jobs) { table(:ci_builds) } + + STATUSES = { created: 0, pending: 1, running: 2, success: 3, + failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze + + before do + projects.create!(id: 123, name: 'gitlab', path: 'gitlab-ce') + pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') + + jobs.create!(id: 1, commit_id: 1, project_id: 123, + stage_idx: 2, stage: 'build', status: :success) + jobs.create!(id: 2, commit_id: 1, project_id: 123, + stage_idx: 2, stage: 'build', status: :success) + jobs.create!(id: 3, commit_id: 1, project_id: 123, + stage_idx: 1, stage: 'test', status: :failed) + jobs.create!(id: 4, commit_id: 1, project_id: 123, + stage_idx: 1, stage: 'test', status: :success) + jobs.create!(id: 5, commit_id: 1, project_id: 123, + stage_idx: 3, stage: 'deploy', status: :pending) + jobs.create!(id: 6, commit_id: 1, project_id: 123, + stage_idx: 3, stage: nil, status: :pending) + end + + it 'correctly migrates builds stages' do + expect(stages.count).to be_zero + + described_class.new.perform(1, 6) + + expect(stages.count).to eq 3 + expect(stages.all.pluck(:name)).to match_array %w[test build deploy] + expect(jobs.where(stage_id: nil)).to be_one + expect(jobs.find_by(stage_id: nil).id).to eq 6 + expect(stages.all.pluck(:status)).to match_array [STATUSES[:success], + STATUSES[:failed], + STATUSES[:pending]] + end + + it 'recovers from unique constraint violation only twice' do + allow(described_class::Migratable::Stage) + .to receive(:find_by).and_return(nil) + + expect(described_class::Migratable::Stage) + .to receive(:find_by).exactly(3).times + + expect { described_class.new.perform(1, 6) } + .to raise_error ActiveRecord::RecordNotUnique + end +end diff --git a/spec/migrations/schedule_build_stage_migration_spec.rb b/spec/migrations/schedule_build_stage_migration_spec.rb new file mode 100644 index 00000000000..06657410173 --- /dev/null +++ b/spec/migrations/schedule_build_stage_migration_spec.rb @@ -0,0 +1,35 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20180212101928_schedule_build_stage_migration') + +describe ScheduleBuildStageMigration, :migration do + let(:projects) { table(:projects) } + let(:pipelines) { table(:ci_pipelines) } + let(:stages) { table(:ci_stages) } + let(:jobs) { table(:ci_builds) } + + before do + stub_const("#{described_class}::BATCH_SIZE", 1) + + projects.create!(id: 123, name: 'gitlab', path: 'gitlab-ce') + pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') + stages.create!(id: 1, project_id: 123, pipeline_id: 1, name: 'test') + + jobs.create!(id: 11, commit_id: 1, project_id: 123, stage_id: nil) + jobs.create!(id: 206, commit_id: 1, project_id: 123, stage_id: nil) + jobs.create!(id: 3413, commit_id: 1, project_id: 123, stage_id: nil) + jobs.create!(id: 4109, commit_id: 1, project_id: 123, stage_id: 1) + end + + it 'schedules delayed background migrations in batches in bulk' do + Sidekiq::Testing.fake! do + Timecop.freeze do + migrate! + + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(5.minutes, 11, 11) + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(10.minutes, 206, 206) + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(15.minutes, 3413, 3413) + expect(BackgroundMigrationWorker.jobs.size).to eq 3 + end + end + end +end |