summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--changelogs/unreleased/feature-gb-pipeline-variable-expressions.yml5
-rw-r--r--db/post_migrate/20180212101828_add_tmp_partial_null_index_to_builds.rb14
-rw-r--r--db/post_migrate/20180212101928_schedule_build_stage_migration.rb29
-rw-r--r--db/post_migrate/20180212102028_remove_tmp_partial_null_index_from_builds.rb14
-rw-r--r--lib/gitlab/background_migration/migrate_build_stage.rb48
-rw-r--r--spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb54
-rw-r--r--spec/migrations/schedule_build_stage_migration_spec.rb35
7 files changed, 199 insertions, 0 deletions
diff --git a/changelogs/unreleased/feature-gb-pipeline-variable-expressions.yml b/changelogs/unreleased/feature-gb-pipeline-variable-expressions.yml
new file mode 100644
index 00000000000..28820649af3
--- /dev/null
+++ b/changelogs/unreleased/feature-gb-pipeline-variable-expressions.yml
@@ -0,0 +1,5 @@
+---
+title: Add catch-up background migration to migrate pipeline stages
+merge_request: 15741
+author:
+type: performance
diff --git a/db/post_migrate/20180212101828_add_tmp_partial_null_index_to_builds.rb b/db/post_migrate/20180212101828_add_tmp_partial_null_index_to_builds.rb
new file mode 100644
index 00000000000..e55e2e6f888
--- /dev/null
+++ b/db/post_migrate/20180212101828_add_tmp_partial_null_index_to_builds.rb
@@ -0,0 +1,14 @@
+class AddTmpPartialNullIndexToBuilds < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ disable_ddl_transaction!
+
+ def up
+ add_concurrent_index(:ci_builds, :id, where: 'stage_id IS NULL',
+ name: 'tmp_id_partial_null_index')
+ end
+
+ def down
+ remove_concurrent_index_by_name(:ci_builds, 'tmp_id_partial_null_index')
+ end
+end
diff --git a/db/post_migrate/20180212101928_schedule_build_stage_migration.rb b/db/post_migrate/20180212101928_schedule_build_stage_migration.rb
new file mode 100644
index 00000000000..df15b2cd9d4
--- /dev/null
+++ b/db/post_migrate/20180212101928_schedule_build_stage_migration.rb
@@ -0,0 +1,29 @@
+class ScheduleBuildStageMigration < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+ MIGRATION = 'MigrateBuildStage'.freeze
+ BATCH_SIZE = 500
+
+ disable_ddl_transaction!
+
+ class Build < ActiveRecord::Base
+ include EachBatch
+ self.table_name = 'ci_builds'
+ end
+
+ def up
+ disable_statement_timeout
+
+ Build.where('stage_id IS NULL').tap do |relation|
+ queue_background_migration_jobs_by_range_at_intervals(relation,
+ MIGRATION,
+ 5.minutes,
+ batch_size: BATCH_SIZE)
+ end
+ end
+
+ def down
+ # noop
+ end
+end
diff --git a/db/post_migrate/20180212102028_remove_tmp_partial_null_index_from_builds.rb b/db/post_migrate/20180212102028_remove_tmp_partial_null_index_from_builds.rb
new file mode 100644
index 00000000000..ed7b1fc72f4
--- /dev/null
+++ b/db/post_migrate/20180212102028_remove_tmp_partial_null_index_from_builds.rb
@@ -0,0 +1,14 @@
+class RemoveTmpPartialNullIndexFromBuilds < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ disable_ddl_transaction!
+
+ def up
+ remove_concurrent_index_by_name(:ci_builds, 'tmp_id_partial_null_index')
+ end
+
+ def down
+ add_concurrent_index(:ci_builds, :id, where: 'stage_id IS NULL',
+ name: 'tmp_id_partial_null_index')
+ end
+end
diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb
new file mode 100644
index 00000000000..8fe4f1a2289
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_build_stage.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Style/Documentation
+
+module Gitlab
+ module BackgroundMigration
+ class MigrateBuildStage
+ module Migratable
+ class Stage < ActiveRecord::Base
+ self.table_name = 'ci_stages'
+ end
+
+ class Build < ActiveRecord::Base
+ self.table_name = 'ci_builds'
+
+ def ensure_stage!(attempts: 2)
+ find_stage || create_stage!
+ rescue ActiveRecord::RecordNotUnique
+ retry if (attempts -= 1) > 0
+ raise
+ end
+
+ def find_stage
+ Stage.find_by(name: self.stage || 'test',
+ pipeline_id: self.commit_id,
+ project_id: self.project_id)
+ end
+
+ def create_stage!
+ Stage.create!(name: self.stage || 'test',
+ pipeline_id: self.commit_id,
+ project_id: self.project_id)
+ end
+ end
+ end
+
+ def perform(start_id, stop_id)
+ stages = Migratable::Build.where('stage_id IS NULL')
+ .where('id BETWEEN ? AND ?', start_id, stop_id)
+ .map { |build| build.ensure_stage! }
+ .compact.map(&:id)
+
+ MigrateBuildStageIdReference.new.perform(start_id, stop_id)
+ MigrateStageStatus.new.perform(stages.min, stages.max)
+ end
+ end
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb b/spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb
new file mode 100644
index 00000000000..e112e9e9e3d
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb
@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::MigrateBuildStage, :migration, schema: 20180212101928 do
+ let(:projects) { table(:projects) }
+ let(:pipelines) { table(:ci_pipelines) }
+ let(:stages) { table(:ci_stages) }
+ let(:jobs) { table(:ci_builds) }
+
+ STATUSES = { created: 0, pending: 1, running: 2, success: 3,
+ failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze
+
+ before do
+ projects.create!(id: 123, name: 'gitlab', path: 'gitlab-ce')
+ pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
+
+ jobs.create!(id: 1, commit_id: 1, project_id: 123,
+ stage_idx: 2, stage: 'build', status: :success)
+ jobs.create!(id: 2, commit_id: 1, project_id: 123,
+ stage_idx: 2, stage: 'build', status: :success)
+ jobs.create!(id: 3, commit_id: 1, project_id: 123,
+ stage_idx: 1, stage: 'test', status: :failed)
+ jobs.create!(id: 4, commit_id: 1, project_id: 123,
+ stage_idx: 1, stage: 'test', status: :success)
+ jobs.create!(id: 5, commit_id: 1, project_id: 123,
+ stage_idx: 3, stage: 'deploy', status: :pending)
+ jobs.create!(id: 6, commit_id: 1, project_id: 123,
+ stage_idx: 3, stage: nil, status: :pending)
+ end
+
+ it 'correctly migrates builds stages' do
+ expect(stages.count).to be_zero
+
+ described_class.new.perform(1, 6)
+
+ expect(stages.count).to eq 3
+ expect(stages.all.pluck(:name)).to match_array %w[test build deploy]
+ expect(jobs.where(stage_id: nil)).to be_one
+ expect(jobs.find_by(stage_id: nil).id).to eq 6
+ expect(stages.all.pluck(:status)).to match_array [STATUSES[:success],
+ STATUSES[:failed],
+ STATUSES[:pending]]
+ end
+
+ it 'recovers from unique constraint violation only twice' do
+ allow(described_class::Migratable::Stage)
+ .to receive(:find_by).and_return(nil)
+
+ expect(described_class::Migratable::Stage)
+ .to receive(:find_by).exactly(3).times
+
+ expect { described_class.new.perform(1, 6) }
+ .to raise_error ActiveRecord::RecordNotUnique
+ end
+end
diff --git a/spec/migrations/schedule_build_stage_migration_spec.rb b/spec/migrations/schedule_build_stage_migration_spec.rb
new file mode 100644
index 00000000000..06657410173
--- /dev/null
+++ b/spec/migrations/schedule_build_stage_migration_spec.rb
@@ -0,0 +1,35 @@
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20180212101928_schedule_build_stage_migration')
+
+describe ScheduleBuildStageMigration, :migration do
+ let(:projects) { table(:projects) }
+ let(:pipelines) { table(:ci_pipelines) }
+ let(:stages) { table(:ci_stages) }
+ let(:jobs) { table(:ci_builds) }
+
+ before do
+ stub_const("#{described_class}::BATCH_SIZE", 1)
+
+ projects.create!(id: 123, name: 'gitlab', path: 'gitlab-ce')
+ pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
+ stages.create!(id: 1, project_id: 123, pipeline_id: 1, name: 'test')
+
+ jobs.create!(id: 11, commit_id: 1, project_id: 123, stage_id: nil)
+ jobs.create!(id: 206, commit_id: 1, project_id: 123, stage_id: nil)
+ jobs.create!(id: 3413, commit_id: 1, project_id: 123, stage_id: nil)
+ jobs.create!(id: 4109, commit_id: 1, project_id: 123, stage_id: 1)
+ end
+
+ it 'schedules delayed background migrations in batches in bulk' do
+ Sidekiq::Testing.fake! do
+ Timecop.freeze do
+ migrate!
+
+ expect(described_class::MIGRATION).to be_scheduled_delayed_migration(5.minutes, 11, 11)
+ expect(described_class::MIGRATION).to be_scheduled_delayed_migration(10.minutes, 206, 206)
+ expect(described_class::MIGRATION).to be_scheduled_delayed_migration(15.minutes, 3413, 3413)
+ expect(BackgroundMigrationWorker.jobs.size).to eq 3
+ end
+ end
+ end
+end