From 6d8963e33657b80821900a60f66d0a51416dc59a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 25 May 2017 15:35:09 +0200 Subject: Add PoC of ActiveRecord migration testing --- config/database.yml.postgresql | 11 ++ .../20170525132202_migrate_pipeline_stages.rb | 14 +++ spec/migrations/migrate_pipeline_stages_spec.rb | 127 +++++++++++++++++++++ spec/migrations_helper.rb | 36 ++++++ 4 files changed, 188 insertions(+) create mode 100644 db/migrate/20170525132202_migrate_pipeline_stages.rb create mode 100644 spec/migrations/migrate_pipeline_stages_spec.rb create mode 100644 spec/migrations_helper.rb diff --git a/config/database.yml.postgresql b/config/database.yml.postgresql index c517a4c0cb8..3dd96015577 100644 --- a/config/database.yml.postgresql +++ b/config/database.yml.postgresql @@ -46,3 +46,14 @@ test: &test username: postgres password: # host: localhost + +# Warning: The database defined as "migrate" might erased. +# Do not set this db to the same as development or production. +migrate: &test + adapter: postgresql + encoding: unicode + database: gitlabhq_migrate + pool: 5 + username: postgres + password: + # host: localhost diff --git a/db/migrate/20170525132202_migrate_pipeline_stages.rb b/db/migrate/20170525132202_migrate_pipeline_stages.rb new file mode 100644 index 00000000000..8f7da8662ec --- /dev/null +++ b/db/migrate/20170525132202_migrate_pipeline_stages.rb @@ -0,0 +1,14 @@ +class MigratePipelineStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + create_table :ci_stages do |t| + t.integer :project_id + t.integer :pipeline_id + t.string :name + t.timestamps + end + end +end diff --git a/spec/migrations/migrate_pipeline_stages_spec.rb b/spec/migrations/migrate_pipeline_stages_spec.rb new file mode 100644 index 00000000000..95c15d53a68 --- /dev/null +++ b/spec/migrations/migrate_pipeline_stages_spec.rb @@ -0,0 +1,127 @@ +require 'migrations_helper' +require Rails.root.join('db', 'migrate', '20170525132202_migrate_pipeline_stages.rb') + +describe MigratePipelineStages, :migration do + ## + # Create tables using schema from which we will migrate stuff. + # + before do + ActiveRecord::Schema.define(version: 20170523091700) do + enable_extension "plpgsql" + enable_extension "pg_trgm" + + create_table "ci_pipelines", force: :cascade do |t| + t.string "ref" + t.string "sha" + t.string "before_sha" + t.datetime "created_at" + t.datetime "updated_at" + t.boolean "tag", default: false + t.text "yaml_errors" + t.datetime "committed_at" + t.integer "project_id" + t.string "status" + t.datetime "started_at" + t.datetime "finished_at" + t.integer "duration" + t.integer "user_id" + t.integer "lock_version" + t.integer "auto_canceled_by_id" + t.integer "pipeline_schedule_id" + end + + add_index "ci_pipelines", ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id", using: :btree + add_index "ci_pipelines", ["pipeline_schedule_id"], name: "index_ci_pipelines_on_pipeline_schedule_id", using: :btree + add_index "ci_pipelines", ["project_id", "ref", "status"], name: "index_ci_pipelines_on_project_id_and_ref_and_status", using: :btree + add_index "ci_pipelines", ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree + add_index "ci_pipelines", ["project_id"], name: "index_ci_pipelines_on_project_id", using: :btree + add_index "ci_pipelines", ["status"], name: "index_ci_pipelines_on_status", using: :btree + add_index "ci_pipelines", ["user_id"], name: "index_ci_pipelines_on_user_id", using: :btree + + create_table "ci_builds", force: :cascade do |t| + t.string "status" + t.datetime "finished_at" + t.text "trace" + t.datetime "created_at" + t.datetime "updated_at" + t.datetime "started_at" + t.integer "runner_id" + t.float "coverage" + t.integer "commit_id" + t.text "commands" + t.string "name" + t.text "options" + t.boolean "allow_failure", default: false, null: false + t.string "stage" + t.integer "trigger_request_id" + t.integer "stage_idx" + t.boolean "tag" + t.string "ref" + t.integer "user_id" + t.string "type" + t.string "target_url" + t.string "description" + t.text "artifacts_file" + t.integer "project_id" + t.text "artifacts_metadata" + t.integer "erased_by_id" + t.datetime "erased_at" + t.datetime "artifacts_expire_at" + t.string "environment" + t.integer "artifacts_size", limit: 8 + t.string "when" + t.text "yaml_variables" + t.datetime "queued_at" + t.string "token" + t.integer "lock_version" + t.string "coverage_regex" + t.integer "auto_canceled_by_id" + t.boolean "retried" + end + + add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree + add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree + add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree + add_index "ci_builds", ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree + add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree + add_index "ci_builds", ["project_id"], name: "index_ci_builds_on_project_id", using: :btree + add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree + add_index "ci_builds", ["status", "type", "runner_id"], name: "index_ci_builds_on_status_and_type_and_runner_id", using: :btree + add_index "ci_builds", ["status"], name: "index_ci_builds_on_status", using: :btree + add_index "ci_builds", ["token"], name: "index_ci_builds_on_token", unique: true, using: :btree + add_index "ci_builds", ["updated_at"], name: "index_ci_builds_on_updated_at", using: :btree + add_index "ci_builds", ["user_id"], name: "index_ci_builds_on_user_id", using: :btree + end + end + + let(:pipeline) do + Class.new(ActiveRecord::Base) do + self.table_name = 'ci_pipelines' + end + end + + let(:build) do + Class.new(ActiveRecord::Base) do + self.table_name = 'ci_builds' + end + end + + let(:stage) do + Class.new(ActiveRecord::Base) do + self.table_name = 'ci_stages' + end + end + + ## + # Create test data + # + before do + pipeline.create!(ref: 'master', sha: 'adf43c3a') + end + + it 'correctly migrates pipeline stages' do + described_class.new.change + + expect(stage.table_exists?).to be true + end +end diff --git a/spec/migrations_helper.rb b/spec/migrations_helper.rb new file mode 100644 index 00000000000..bbb810556da --- /dev/null +++ b/spec/migrations_helper.rb @@ -0,0 +1,36 @@ +require File.expand_path("../../config/environment", __FILE__) +require 'rspec/rails' +require 'shoulda/matchers' + +ActiveRecord::Base.establish_connection(:migrate) + +RSpec.configure do |config| + config.mock_with :rspec + config.verbose_retry = true + config.display_try_failure_messages = true + config.use_transactional_fixtures = true + config.infer_spec_type_from_file_location! + config.raise_errors_for_deprecations! + + config.around(:each, :migration) do |example| + ActiveRecord::Tasks::DatabaseTasks.purge_current + + example.run + + ActiveRecord::Tasks::DatabaseTasks.purge_current + end + + config.around(:each, :redis) do |example| + Gitlab::Redis.with(&:flushall) + Sidekiq.redis(&:flushall) + + example.run + + Gitlab::Redis.with(&:flushall) + Sidekiq.redis(&:flushall) + end +end + + +puts "Rails environment: #{Rails.env}" +puts "Database connection: #{ActiveRecord::Base.connection_config[:database]}" -- cgit v1.2.1 From acc22a8422cd1471819510aa375c455b5ea009c5 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 25 May 2017 22:43:16 +0200 Subject: Simplify migrations for specific database schema --- config/database.yml.postgresql | 11 -- .../20170525132202_migrate_pipeline_stages.rb | 2 +- db/schema.rb | 10 +- spec/migrations/migrate_pipeline_stages_spec.rb | 122 ++------------------- spec/migrations_helper.rb | 36 ------ spec/spec_helper.rb | 10 ++ 6 files changed, 29 insertions(+), 162 deletions(-) delete mode 100644 spec/migrations_helper.rb diff --git a/config/database.yml.postgresql b/config/database.yml.postgresql index 3dd96015577..c517a4c0cb8 100644 --- a/config/database.yml.postgresql +++ b/config/database.yml.postgresql @@ -46,14 +46,3 @@ test: &test username: postgres password: # host: localhost - -# Warning: The database defined as "migrate" might erased. -# Do not set this db to the same as development or production. -migrate: &test - adapter: postgresql - encoding: unicode - database: gitlabhq_migrate - pool: 5 - username: postgres - password: - # host: localhost diff --git a/db/migrate/20170525132202_migrate_pipeline_stages.rb b/db/migrate/20170525132202_migrate_pipeline_stages.rb index 8f7da8662ec..a9fe00ebf72 100644 --- a/db/migrate/20170525132202_migrate_pipeline_stages.rb +++ b/db/migrate/20170525132202_migrate_pipeline_stages.rb @@ -8,7 +8,7 @@ class MigratePipelineStages < ActiveRecord::Migration t.integer :project_id t.integer :pipeline_id t.string :name - t.timestamps + t.timestamps null: true end end end diff --git a/db/schema.rb b/db/schema.rb index ca33c8cc2a2..581060f62fc 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170523091700) do +ActiveRecord::Schema.define(version: 20170525132202) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -325,6 +325,14 @@ ActiveRecord::Schema.define(version: 20170523091700) do add_index "ci_runners", ["locked"], name: "index_ci_runners_on_locked", using: :btree add_index "ci_runners", ["token"], name: "index_ci_runners_on_token", using: :btree + create_table "ci_stages", force: :cascade do |t| + t.integer "project_id" + t.integer "pipeline_id" + t.string "name" + t.datetime "created_at" + t.datetime "updated_at" + end + create_table "ci_trigger_requests", force: :cascade do |t| t.integer "trigger_id", null: false t.text "variables" diff --git a/spec/migrations/migrate_pipeline_stages_spec.rb b/spec/migrations/migrate_pipeline_stages_spec.rb index 95c15d53a68..038e5d7fc6b 100644 --- a/spec/migrations/migrate_pipeline_stages_spec.rb +++ b/spec/migrations/migrate_pipeline_stages_spec.rb @@ -1,127 +1,23 @@ -require 'migrations_helper' +require 'spec_helper' require Rails.root.join('db', 'migrate', '20170525132202_migrate_pipeline_stages.rb') -describe MigratePipelineStages, :migration do - ## - # Create tables using schema from which we will migrate stuff. - # - before do - ActiveRecord::Schema.define(version: 20170523091700) do - enable_extension "plpgsql" - enable_extension "pg_trgm" - - create_table "ci_pipelines", force: :cascade do |t| - t.string "ref" - t.string "sha" - t.string "before_sha" - t.datetime "created_at" - t.datetime "updated_at" - t.boolean "tag", default: false - t.text "yaml_errors" - t.datetime "committed_at" - t.integer "project_id" - t.string "status" - t.datetime "started_at" - t.datetime "finished_at" - t.integer "duration" - t.integer "user_id" - t.integer "lock_version" - t.integer "auto_canceled_by_id" - t.integer "pipeline_schedule_id" - end - - add_index "ci_pipelines", ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id", using: :btree - add_index "ci_pipelines", ["pipeline_schedule_id"], name: "index_ci_pipelines_on_pipeline_schedule_id", using: :btree - add_index "ci_pipelines", ["project_id", "ref", "status"], name: "index_ci_pipelines_on_project_id_and_ref_and_status", using: :btree - add_index "ci_pipelines", ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree - add_index "ci_pipelines", ["project_id"], name: "index_ci_pipelines_on_project_id", using: :btree - add_index "ci_pipelines", ["status"], name: "index_ci_pipelines_on_status", using: :btree - add_index "ci_pipelines", ["user_id"], name: "index_ci_pipelines_on_user_id", using: :btree - - create_table "ci_builds", force: :cascade do |t| - t.string "status" - t.datetime "finished_at" - t.text "trace" - t.datetime "created_at" - t.datetime "updated_at" - t.datetime "started_at" - t.integer "runner_id" - t.float "coverage" - t.integer "commit_id" - t.text "commands" - t.string "name" - t.text "options" - t.boolean "allow_failure", default: false, null: false - t.string "stage" - t.integer "trigger_request_id" - t.integer "stage_idx" - t.boolean "tag" - t.string "ref" - t.integer "user_id" - t.string "type" - t.string "target_url" - t.string "description" - t.text "artifacts_file" - t.integer "project_id" - t.text "artifacts_metadata" - t.integer "erased_by_id" - t.datetime "erased_at" - t.datetime "artifacts_expire_at" - t.string "environment" - t.integer "artifacts_size", limit: 8 - t.string "when" - t.text "yaml_variables" - t.datetime "queued_at" - t.string "token" - t.integer "lock_version" - t.string "coverage_regex" - t.integer "auto_canceled_by_id" - t.boolean "retried" - end - - add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree - add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree - add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree - add_index "ci_builds", ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree - add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree - add_index "ci_builds", ["project_id"], name: "index_ci_builds_on_project_id", using: :btree - add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree - add_index "ci_builds", ["status", "type", "runner_id"], name: "index_ci_builds_on_status_and_type_and_runner_id", using: :btree - add_index "ci_builds", ["status"], name: "index_ci_builds_on_status", using: :btree - add_index "ci_builds", ["token"], name: "index_ci_builds_on_token", unique: true, using: :btree - add_index "ci_builds", ["updated_at"], name: "index_ci_builds_on_updated_at", using: :btree - add_index "ci_builds", ["user_id"], name: "index_ci_builds_on_user_id", using: :btree - end - end - - let(:pipeline) do - Class.new(ActiveRecord::Base) do - self.table_name = 'ci_pipelines' - end - end - - let(:build) do - Class.new(ActiveRecord::Base) do - self.table_name = 'ci_builds' - end - end - - let(:stage) do - Class.new(ActiveRecord::Base) do - self.table_name = 'ci_stages' - end +describe MigratePipelineStages, :migration, schema: 20170523091700 do + def table(name) + Class.new(ActiveRecord::Base) { self.table_name = name } end ## # Create test data # before do - pipeline.create!(ref: 'master', sha: 'adf43c3a') + table(:ci_pipelines).create!(ref: 'master', sha: 'adf43c3a') end it 'correctly migrates pipeline stages' do - described_class.new.change + expect(ActiveRecord::Base.connection.table_exists?('ci_stages')).to eq false + + described_class.new.migrate(:up) - expect(stage.table_exists?).to be true + expect(ActiveRecord::Base.connection.table_exists?('ci_stages')).to eq true end end diff --git a/spec/migrations_helper.rb b/spec/migrations_helper.rb deleted file mode 100644 index bbb810556da..00000000000 --- a/spec/migrations_helper.rb +++ /dev/null @@ -1,36 +0,0 @@ -require File.expand_path("../../config/environment", __FILE__) -require 'rspec/rails' -require 'shoulda/matchers' - -ActiveRecord::Base.establish_connection(:migrate) - -RSpec.configure do |config| - config.mock_with :rspec - config.verbose_retry = true - config.display_try_failure_messages = true - config.use_transactional_fixtures = true - config.infer_spec_type_from_file_location! - config.raise_errors_for_deprecations! - - config.around(:each, :migration) do |example| - ActiveRecord::Tasks::DatabaseTasks.purge_current - - example.run - - ActiveRecord::Tasks::DatabaseTasks.purge_current - end - - config.around(:each, :redis) do |example| - Gitlab::Redis.with(&:flushall) - Sidekiq.redis(&:flushall) - - example.run - - Gitlab::Redis.with(&:flushall) - Sidekiq.redis(&:flushall) - end -end - - -puts "Rails environment: #{Rails.env}" -puts "Database connection: #{ActiveRecord::Base.connection_config[:database]}" diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 51571ddebe9..992c0c29c72 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -92,6 +92,16 @@ RSpec.configure do |config| Gitlab::Redis.with(&:flushall) Sidekiq.redis(&:flushall) end + + config.around(:example, migration: true) do |example| + schema_version = example.metadata[:schema] + migrations_paths = ActiveRecord::Migrator.migrations_paths + ActiveRecord::Migrator.migrate(migrations_paths, schema_version) + + example.run + + ActiveRecord::Migration.maintain_test_schema! + end end FactoryGirl::SyntaxRunner.class_eval do -- cgit v1.2.1 From f7aba153164618522232c912eb833bd7596f5f53 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 26 May 2017 10:29:12 +0200 Subject: Make migration specs by using migrator in around hook --- spec/migrations/migrate_pipeline_stages_spec.rb | 17 +++++++++++++++-- spec/spec_helper.rb | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/spec/migrations/migrate_pipeline_stages_spec.rb b/spec/migrations/migrate_pipeline_stages_spec.rb index 038e5d7fc6b..d6d26ab52ad 100644 --- a/spec/migrations/migrate_pipeline_stages_spec.rb +++ b/spec/migrations/migrate_pipeline_stages_spec.rb @@ -2,10 +2,23 @@ require 'spec_helper' require Rails.root.join('db', 'migrate', '20170525132202_migrate_pipeline_stages.rb') describe MigratePipelineStages, :migration, schema: 20170523091700 do + ## + # TODO, extract to migrations helper + # def table(name) Class.new(ActiveRecord::Base) { self.table_name = name } end + def migrations_paths + ActiveRecord::Migrator.migrations_paths + end + + def migrate! + ActiveRecord::Migrator.up(migrations_paths) do |migration| + migration.name == described_class.name + end + end + ## # Create test data # @@ -13,10 +26,10 @@ describe MigratePipelineStages, :migration, schema: 20170523091700 do table(:ci_pipelines).create!(ref: 'master', sha: 'adf43c3a') end - it 'correctly migrates pipeline stages' do + it 'correctly migrates pipeline stages' do |migration, meta| expect(ActiveRecord::Base.connection.table_exists?('ci_stages')).to eq false - described_class.new.migrate(:up) + migrate! expect(ActiveRecord::Base.connection.table_exists?('ci_stages')).to eq true end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 992c0c29c72..9ed705a5267 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -100,7 +100,7 @@ RSpec.configure do |config| example.run - ActiveRecord::Migration.maintain_test_schema! + ActiveRecord::Migrator.migrate(migrations_paths) end end -- cgit v1.2.1 From 0f9fbae78a51f7bf4df50d96060087e1cf903b05 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 26 May 2017 10:36:01 +0200 Subject: Add migrations helpers to simplify writing migration specs --- spec/migrations/migrate_pipeline_stages_spec.rb | 17 ----------------- spec/spec_helper.rb | 6 +++--- spec/support/migrations_helpers.rb | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 20 deletions(-) create mode 100644 spec/support/migrations_helpers.rb diff --git a/spec/migrations/migrate_pipeline_stages_spec.rb b/spec/migrations/migrate_pipeline_stages_spec.rb index d6d26ab52ad..4a469174b9e 100644 --- a/spec/migrations/migrate_pipeline_stages_spec.rb +++ b/spec/migrations/migrate_pipeline_stages_spec.rb @@ -2,23 +2,6 @@ require 'spec_helper' require Rails.root.join('db', 'migrate', '20170525132202_migrate_pipeline_stages.rb') describe MigratePipelineStages, :migration, schema: 20170523091700 do - ## - # TODO, extract to migrations helper - # - def table(name) - Class.new(ActiveRecord::Base) { self.table_name = name } - end - - def migrations_paths - ActiveRecord::Migrator.migrations_paths - end - - def migrate! - ActiveRecord::Migrator.up(migrations_paths) do |migration| - migration.name == described_class.name - end - end - ## # Create test data # diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 9ed705a5267..2c1c252be37 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -52,6 +52,7 @@ RSpec.configure do |config| config.include StubGitlabCalls config.include StubGitlabData config.include ApiHelpers, :api + config.include MigrationsHelpers, :migration config.infer_spec_type_from_file_location! @@ -94,9 +95,8 @@ RSpec.configure do |config| end config.around(:example, migration: true) do |example| - schema_version = example.metadata[:schema] - migrations_paths = ActiveRecord::Migrator.migrations_paths - ActiveRecord::Migrator.migrate(migrations_paths, schema_version) + ActiveRecord::Migrator + .migrate(migrations_paths, example.metadata.fetch(:schema)) example.run diff --git a/spec/support/migrations_helpers.rb b/spec/support/migrations_helpers.rb new file mode 100644 index 00000000000..965b00b5731 --- /dev/null +++ b/spec/support/migrations_helpers.rb @@ -0,0 +1,15 @@ +module MigrationsHelpers + def table(name) + Class.new(ActiveRecord::Base) { self.table_name = name } + end + + def migrations_paths + ActiveRecord::Migrator.migrations_paths + end + + def migrate! + ActiveRecord::Migrator.up(migrations_paths) do |migration| + migration.name == described_class.name + end + end +end -- cgit v1.2.1 From 918ababba6fce694c61d54bb2ff6983a886f696a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 26 May 2017 14:46:45 +0200 Subject: Add pipeline stages post deployment migration --- .../20170525132202_create_pipeline_stages.rb | 14 +++++++ .../20170525132202_migrate_pipeline_stages.rb | 14 ------- .../20170526101042_migrate_pipeline_stages.rb | 43 +++++++++++++++++++++ db/schema.rb | 5 ++- spec/migrations/migrate_pipeline_stages_spec.rb | 44 ++++++++++++++++++---- spec/spec_helper.rb | 4 +- spec/support/db_cleaner.rb | 4 ++ spec/support/migrations_helpers.rb | 4 ++ 8 files changed, 108 insertions(+), 24 deletions(-) create mode 100644 db/migrate/20170525132202_create_pipeline_stages.rb delete mode 100644 db/migrate/20170525132202_migrate_pipeline_stages.rb create mode 100644 db/post_migrate/20170526101042_migrate_pipeline_stages.rb diff --git a/db/migrate/20170525132202_create_pipeline_stages.rb b/db/migrate/20170525132202_create_pipeline_stages.rb new file mode 100644 index 00000000000..c562df27357 --- /dev/null +++ b/db/migrate/20170525132202_create_pipeline_stages.rb @@ -0,0 +1,14 @@ +class CreatePipelineStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + create_table :ci_stages do |t| + t.integer :project_id + t.integer :pipeline_id + t.string :name + t.timestamps null: true + end + end +end diff --git a/db/migrate/20170525132202_migrate_pipeline_stages.rb b/db/migrate/20170525132202_migrate_pipeline_stages.rb deleted file mode 100644 index a9fe00ebf72..00000000000 --- a/db/migrate/20170525132202_migrate_pipeline_stages.rb +++ /dev/null @@ -1,14 +0,0 @@ -class MigratePipelineStages < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - def change - create_table :ci_stages do |t| - t.integer :project_id - t.integer :pipeline_id - t.string :name - t.timestamps null: true - end - end -end diff --git a/db/post_migrate/20170526101042_migrate_pipeline_stages.rb b/db/post_migrate/20170526101042_migrate_pipeline_stages.rb new file mode 100644 index 00000000000..12ad0db3a60 --- /dev/null +++ b/db/post_migrate/20170526101042_migrate_pipeline_stages.rb @@ -0,0 +1,43 @@ +class MigratePipelineStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + disable_statement_timeout + + execute <<-SQL.strip_heredoc + INSERT INTO ci_stages (project_id, pipeline_id, name) + SELECT project_id, commit_id, stage FROM ci_builds + WHERE stage IS NOT NULL + GROUP BY project_id, commit_id, stage, stage_idx + ORDER BY stage_idx + SQL + + add_concurrent_index(:ci_stages, [:pipeline_id, :name]) + + add_column(:ci_builds, :stage_id, :integer) + + stage_id = Arel.sql('(SELECT id FROM ci_stages ' \ + 'WHERE ci_stages.pipeline_id = ci_builds.commit_id ' \ + 'AND ci_stages.name = ci_builds.stage)') + update_column_in_batches(:ci_builds, :stage_id, stage_id) + + # add_concurrent_foreign_key :ci_stages, :projects, column: :project_id, on_delete: :cascade + # add_concurrent_foreign_key :ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade + end + + def down + execute('TRUNCATE TABLE ci_stages') + + if column_exists?(:ci_builds, :stage_id) + remove_column(:ci_builds, :stage_id) + end + + if index_exists?(:ci_stages, [:pipeline_id, :name]) + remove_index(:ci_stages, [:pipeline_id, :name]) + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 581060f62fc..f1df8d6bd8c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170525132202) do +ActiveRecord::Schema.define(version: 20170526101042) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -233,6 +233,7 @@ ActiveRecord::Schema.define(version: 20170525132202) do t.string "coverage_regex" t.integer "auto_canceled_by_id" t.boolean "retried" + t.integer "stage_id" end add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree @@ -333,6 +334,8 @@ ActiveRecord::Schema.define(version: 20170525132202) do t.datetime "updated_at" end + add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", using: :btree + create_table "ci_trigger_requests", force: :cascade do |t| t.integer "trigger_id", null: false t.text "variables" diff --git a/spec/migrations/migrate_pipeline_stages_spec.rb b/spec/migrations/migrate_pipeline_stages_spec.rb index 4a469174b9e..9cf13a1dabc 100644 --- a/spec/migrations/migrate_pipeline_stages_spec.rb +++ b/spec/migrations/migrate_pipeline_stages_spec.rb @@ -1,19 +1,49 @@ require 'spec_helper' -require Rails.root.join('db', 'migrate', '20170525132202_migrate_pipeline_stages.rb') +require Rails.root.join('db', 'post_migrate', '20170526101042_migrate_pipeline_stages.rb') -describe MigratePipelineStages, :migration, schema: 20170523091700 do +describe MigratePipelineStages, :migration, schema: 20170525132202 do ## - # Create test data + # Create test data - pipeline and CI/CD jobs. # + + let(:jobs) { table(:ci_builds) } + let(:stages) { table(:ci_stages) } + let(:pipelines) { table(:ci_pipelines) } + before do - table(:ci_pipelines).create!(ref: 'master', sha: 'adf43c3a') + # Create CI/CD pipelines + # + pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') + pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb') + + # Create CI/CD jobs + # + jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') + jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') + jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test') + jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test') + jobs.create!(id: 5, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy') + jobs.create!(id: 6, commit_id: 2, project_id: 456, stage_idx: 3, stage: 'deploy') + jobs.create!(id: 7, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2') + jobs.create!(id: 8, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') + jobs.create!(id: 9, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') + jobs.create!(id: 10, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2') + jobs.create!(id: 11, commit_id: 3, project_id: 789, stage_idx: 3, stage: 'deploy') end - it 'correctly migrates pipeline stages' do |migration, meta| - expect(ActiveRecord::Base.connection.table_exists?('ci_stages')).to eq false + it 'correctly migrates pipeline stages' do + expect(stages.count).to be_zero migrate! - expect(ActiveRecord::Base.connection.table_exists?('ci_stages')).to eq true + expect(stages.count).to eq 7 + expect(stages.all.pluck(:name)) + .to match_array %w[test build deploy test:1 test:2 deploy deploy] + expect(stages.where(pipeline_id: 1).order(:id).pluck(:name)) + .to eq %w[test build deploy] + expect(stages.where(pipeline_id: 2).order(:id).pluck(:name)) + .to eq %w[test:1 test:2 deploy] + + expect(jobs.where(stage_id: nil)).to be_empty end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 2c1c252be37..b4ece5c60c1 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -95,8 +95,8 @@ RSpec.configure do |config| end config.around(:example, migration: true) do |example| - ActiveRecord::Migrator - .migrate(migrations_paths, example.metadata.fetch(:schema)) + schema_version = example.metadata.fetch(:schema) + ActiveRecord::Migrator.migrate(migrations_paths, schema_version) example.run diff --git a/spec/support/db_cleaner.rb b/spec/support/db_cleaner.rb index 6f31828b825..7f5769209bb 100644 --- a/spec/support/db_cleaner.rb +++ b/spec/support/db_cleaner.rb @@ -19,6 +19,10 @@ RSpec.configure do |config| DatabaseCleaner.strategy = :truncation end + config.before(:each, :migration) do + DatabaseCleaner.strategy = :truncation + end + config.before(:each) do DatabaseCleaner.start end diff --git a/spec/support/migrations_helpers.rb b/spec/support/migrations_helpers.rb index 965b00b5731..ee17d1a40b7 100644 --- a/spec/support/migrations_helpers.rb +++ b/spec/support/migrations_helpers.rb @@ -7,6 +7,10 @@ module MigrationsHelpers ActiveRecord::Migrator.migrations_paths end + def table_exists?(name) + ActiveRecord::Base.connection.table_exists?(name) + end + def migrate! ActiveRecord::Migrator.up(migrations_paths) do |migration| migration.name == described_class.name -- cgit v1.2.1 From 56681cf7954eeeba227b73cfeb163eb4e51dc96f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 26 May 2017 20:50:43 +0200 Subject: Use a separate migration to create pipeline stages index --- db/post_migrate/20170526101042_migrate_pipeline_stages.rb | 10 ---------- .../20170526180645_create_index_in_pipeline_stages.rb | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 10 deletions(-) create mode 100644 db/post_migrate/20170526180645_create_index_in_pipeline_stages.rb diff --git a/db/post_migrate/20170526101042_migrate_pipeline_stages.rb b/db/post_migrate/20170526101042_migrate_pipeline_stages.rb index 12ad0db3a60..9ef351dff93 100644 --- a/db/post_migrate/20170526101042_migrate_pipeline_stages.rb +++ b/db/post_migrate/20170526101042_migrate_pipeline_stages.rb @@ -16,8 +16,6 @@ class MigratePipelineStages < ActiveRecord::Migration ORDER BY stage_idx SQL - add_concurrent_index(:ci_stages, [:pipeline_id, :name]) - add_column(:ci_builds, :stage_id, :integer) stage_id = Arel.sql('(SELECT id FROM ci_stages ' \ @@ -31,13 +29,5 @@ class MigratePipelineStages < ActiveRecord::Migration def down execute('TRUNCATE TABLE ci_stages') - - if column_exists?(:ci_builds, :stage_id) - remove_column(:ci_builds, :stage_id) - end - - if index_exists?(:ci_stages, [:pipeline_id, :name]) - remove_index(:ci_stages, [:pipeline_id, :name]) - end end end diff --git a/db/post_migrate/20170526180645_create_index_in_pipeline_stages.rb b/db/post_migrate/20170526180645_create_index_in_pipeline_stages.rb new file mode 100644 index 00000000000..d049f87578a --- /dev/null +++ b/db/post_migrate/20170526180645_create_index_in_pipeline_stages.rb @@ -0,0 +1,15 @@ +class CreateIndexInPipelineStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index(:ci_stages, [:pipeline_id, :name]) + end + + def down + remove_index(:ci_stages, [:pipeline_id, :name]) + end +end -- cgit v1.2.1 From b8c39649296049e402f8917b2b8a6123a302e672 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 26 May 2017 20:57:36 +0200 Subject: Add stage_id column to ci_builds in a separate migration --- db/migrate/20170526185602_add_stage_id_to_ci_builds.rb | 9 +++++++++ db/post_migrate/20170526101042_migrate_pipeline_stages.rb | 2 -- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 db/migrate/20170526185602_add_stage_id_to_ci_builds.rb diff --git a/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb b/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb new file mode 100644 index 00000000000..0788f15f7c5 --- /dev/null +++ b/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb @@ -0,0 +1,9 @@ +class AddStageIdToCiBuilds < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :ci_builds, :stage_id, :integer + end +end diff --git a/db/post_migrate/20170526101042_migrate_pipeline_stages.rb b/db/post_migrate/20170526101042_migrate_pipeline_stages.rb index 9ef351dff93..fdbefd59edf 100644 --- a/db/post_migrate/20170526101042_migrate_pipeline_stages.rb +++ b/db/post_migrate/20170526101042_migrate_pipeline_stages.rb @@ -16,8 +16,6 @@ class MigratePipelineStages < ActiveRecord::Migration ORDER BY stage_idx SQL - add_column(:ci_builds, :stage_id, :integer) - stage_id = Arel.sql('(SELECT id FROM ci_stages ' \ 'WHERE ci_stages.pipeline_id = ci_builds.commit_id ' \ 'AND ci_stages.name = ci_builds.stage)') -- cgit v1.2.1 From 33a0fd995bc00bc894243b0d7b2fda0367ae8b4c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 26 May 2017 21:39:32 +0200 Subject: Migrate build stage reference in a separate migration --- .../20170526101042_migrate_pipeline_stages.rb | 8 ---- ...20170526185921_migrate_build_stage_reference.rb | 21 ++++++++ db/schema.rb | 2 +- .../migrate_build_stage_reference_spec.rb | 56 ++++++++++++++++++++++ spec/migrations/migrate_pipeline_stages_spec.rb | 2 - 5 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 db/post_migrate/20170526185921_migrate_build_stage_reference.rb create mode 100644 spec/migrations/migrate_build_stage_reference_spec.rb diff --git a/db/post_migrate/20170526101042_migrate_pipeline_stages.rb b/db/post_migrate/20170526101042_migrate_pipeline_stages.rb index fdbefd59edf..05e095f07cb 100644 --- a/db/post_migrate/20170526101042_migrate_pipeline_stages.rb +++ b/db/post_migrate/20170526101042_migrate_pipeline_stages.rb @@ -15,14 +15,6 @@ class MigratePipelineStages < ActiveRecord::Migration GROUP BY project_id, commit_id, stage, stage_idx ORDER BY stage_idx SQL - - stage_id = Arel.sql('(SELECT id FROM ci_stages ' \ - 'WHERE ci_stages.pipeline_id = ci_builds.commit_id ' \ - 'AND ci_stages.name = ci_builds.stage)') - update_column_in_batches(:ci_builds, :stage_id, stage_id) - - # add_concurrent_foreign_key :ci_stages, :projects, column: :project_id, on_delete: :cascade - # add_concurrent_foreign_key :ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade end def down diff --git a/db/post_migrate/20170526185921_migrate_build_stage_reference.rb b/db/post_migrate/20170526185921_migrate_build_stage_reference.rb new file mode 100644 index 00000000000..8f453b8cd82 --- /dev/null +++ b/db/post_migrate/20170526185921_migrate_build_stage_reference.rb @@ -0,0 +1,21 @@ +class MigrateBuildStageReference < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + disable_statement_timeout + + stage_id = Arel.sql('(SELECT id FROM ci_stages ' \ + 'WHERE ci_stages.pipeline_id = ci_builds.commit_id ' \ + 'AND ci_stages.name = ci_builds.stage)') + + update_column_in_batches(:ci_builds, :stage_id, stage_id) + end + + def down + disable_statement_timeout + + update_column_in_batches(:ci_builds, :stage_id, nil) + end +end diff --git a/db/schema.rb b/db/schema.rb index f1df8d6bd8c..87e16e2f69c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170526101042) do +ActiveRecord::Schema.define(version: 20170526185921) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" diff --git a/spec/migrations/migrate_build_stage_reference_spec.rb b/spec/migrations/migrate_build_stage_reference_spec.rb new file mode 100644 index 00000000000..979f13a1398 --- /dev/null +++ b/spec/migrations/migrate_build_stage_reference_spec.rb @@ -0,0 +1,56 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20170526185921_migrate_build_stage_reference.rb') + +describe MigrateBuildStageReference, :migration, schema: 20170526185602 do + ## + # Create test data - pipeline and CI/CD jobs. + # + + let(:jobs) { table(:ci_builds) } + let(:stages) { table(:ci_stages) } + let(:pipelines) { table(:ci_pipelines) } + + before do + # Create CI/CD pipelines + # + pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') + pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb') + + # Create CI/CD jobs + # + jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') + jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') + jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test') + jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy') + jobs.create!(id: 5, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2') + jobs.create!(id: 6, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') + jobs.create!(id: 7, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') + jobs.create!(id: 8, commit_id: 3, project_id: 789, stage_idx: 3, stage: 'deploy') + + # Create CI/CD stages + # + stages.create(id: 101, pipeline_id: 1, project_id: 123, name: 'test') + stages.create(id: 102, pipeline_id: 1, project_id: 123, name: 'build') + stages.create(id: 103, pipeline_id: 1, project_id: 123, name: 'deploy') + stages.create(id: 104, pipeline_id: 2, project_id: 456, name: 'test:1') + stages.create(id: 105, pipeline_id: 2, project_id: 456, name: 'test:2') + stages.create(id: 106, pipeline_id: 2, project_id: 456, name: 'deploy') + end + + it 'correctly migrate build stage references' do + expect(jobs.where(stage_id: nil).count).to eq 8 + + migrate! + + expect(jobs.where(stage_id: nil).count).to eq 1 + + expect(jobs.find(1).stage_id).to eq 102 + expect(jobs.find(2).stage_id).to eq 102 + expect(jobs.find(3).stage_id).to eq 101 + expect(jobs.find(4).stage_id).to eq 103 + expect(jobs.find(5).stage_id).to eq 105 + expect(jobs.find(6).stage_id).to eq 104 + expect(jobs.find(7).stage_id).to eq 104 + expect(jobs.find(8).stage_id).to eq nil + end +end diff --git a/spec/migrations/migrate_pipeline_stages_spec.rb b/spec/migrations/migrate_pipeline_stages_spec.rb index 9cf13a1dabc..6887fcd26c3 100644 --- a/spec/migrations/migrate_pipeline_stages_spec.rb +++ b/spec/migrations/migrate_pipeline_stages_spec.rb @@ -43,7 +43,5 @@ describe MigratePipelineStages, :migration, schema: 20170525132202 do .to eq %w[test build deploy] expect(stages.where(pipeline_id: 2).order(:id).pluck(:name)) .to eq %w[test:1 test:2 deploy] - - expect(jobs.where(stage_id: nil)).to be_empty end end -- cgit v1.2.1 From 23c93490824bbdf335ef416271d521f65ed3b964 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 26 May 2017 21:49:47 +0200 Subject: Create foreign keys for pipeline stages in migration --- ...526190708_create_foreign_keys_for_pipeline_stages.rb | 17 +++++++++++++++++ db/schema.rb | 4 +++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb diff --git a/db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb b/db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb new file mode 100644 index 00000000000..40060d12d74 --- /dev/null +++ b/db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb @@ -0,0 +1,17 @@ +class CreateForeignKeysForPipelineStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :ci_stages, :projects, column: :project_id, on_delete: :cascade + add_concurrent_foreign_key :ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade + end + + def down + remove_foreign_key :ci_stages, column: :project_id + remove_foreign_key :ci_builds, column: :stage_id + end +end diff --git a/db/schema.rb b/db/schema.rb index 87e16e2f69c..f3bd9c26fb4 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170526185921) do +ActiveRecord::Schema.define(version: 20170526190708) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -1429,10 +1429,12 @@ ActiveRecord::Schema.define(version: 20170526185921) do add_foreign_key "boards", "projects" add_foreign_key "chat_teams", "namespaces", on_delete: :cascade add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify + add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade add_foreign_key "ci_pipeline_schedules", "projects", name: "fk_8ead60fcc4", on_delete: :cascade add_foreign_key "ci_pipeline_schedules", "users", column: "owner_id", name: "fk_9ea99f58d2", on_delete: :nullify add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify + add_foreign_key "ci_stages", "projects", name: "fk_2360681d1d", on_delete: :cascade add_foreign_key "ci_trigger_requests", "ci_triggers", column: "trigger_id", name: "fk_b8ec8b7245", on_delete: :cascade add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade add_foreign_key "ci_variables", "projects", name: "fk_ada5eb64b3", on_delete: :cascade -- cgit v1.2.1 From ff61e2b776badef7cd614e697a7eac20c63534eb Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 26 May 2017 22:33:50 +0200 Subject: Reorder pipeline stages-related migrations --- .../20170526101042_migrate_pipeline_stages.rb | 23 ---------------------- ...170526180645_create_index_in_pipeline_stages.rb | 15 -------------- .../20170526185842_migrate_pipeline_stages.rb | 23 ++++++++++++++++++++++ ...170526190948_create_index_in_pipeline_stages.rb | 15 ++++++++++++++ db/schema.rb | 2 +- spec/migrations/migrate_pipeline_stages_spec.rb | 4 ++-- spec/spec_helper.rb | 12 ++++++----- 7 files changed, 48 insertions(+), 46 deletions(-) delete mode 100644 db/post_migrate/20170526101042_migrate_pipeline_stages.rb delete mode 100644 db/post_migrate/20170526180645_create_index_in_pipeline_stages.rb create mode 100644 db/post_migrate/20170526185842_migrate_pipeline_stages.rb create mode 100644 db/post_migrate/20170526190948_create_index_in_pipeline_stages.rb diff --git a/db/post_migrate/20170526101042_migrate_pipeline_stages.rb b/db/post_migrate/20170526101042_migrate_pipeline_stages.rb deleted file mode 100644 index 05e095f07cb..00000000000 --- a/db/post_migrate/20170526101042_migrate_pipeline_stages.rb +++ /dev/null @@ -1,23 +0,0 @@ -class MigratePipelineStages < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - disable_ddl_transaction! - - def up - disable_statement_timeout - - execute <<-SQL.strip_heredoc - INSERT INTO ci_stages (project_id, pipeline_id, name) - SELECT project_id, commit_id, stage FROM ci_builds - WHERE stage IS NOT NULL - GROUP BY project_id, commit_id, stage, stage_idx - ORDER BY stage_idx - SQL - end - - def down - execute('TRUNCATE TABLE ci_stages') - end -end diff --git a/db/post_migrate/20170526180645_create_index_in_pipeline_stages.rb b/db/post_migrate/20170526180645_create_index_in_pipeline_stages.rb deleted file mode 100644 index d049f87578a..00000000000 --- a/db/post_migrate/20170526180645_create_index_in_pipeline_stages.rb +++ /dev/null @@ -1,15 +0,0 @@ -class CreateIndexInPipelineStages < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - disable_ddl_transaction! - - def up - add_concurrent_index(:ci_stages, [:pipeline_id, :name]) - end - - def down - remove_index(:ci_stages, [:pipeline_id, :name]) - end -end diff --git a/db/post_migrate/20170526185842_migrate_pipeline_stages.rb b/db/post_migrate/20170526185842_migrate_pipeline_stages.rb new file mode 100644 index 00000000000..05e095f07cb --- /dev/null +++ b/db/post_migrate/20170526185842_migrate_pipeline_stages.rb @@ -0,0 +1,23 @@ +class MigratePipelineStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + disable_statement_timeout + + execute <<-SQL.strip_heredoc + INSERT INTO ci_stages (project_id, pipeline_id, name) + SELECT project_id, commit_id, stage FROM ci_builds + WHERE stage IS NOT NULL + GROUP BY project_id, commit_id, stage, stage_idx + ORDER BY stage_idx + SQL + end + + def down + execute('TRUNCATE TABLE ci_stages') + end +end diff --git a/db/post_migrate/20170526190948_create_index_in_pipeline_stages.rb b/db/post_migrate/20170526190948_create_index_in_pipeline_stages.rb new file mode 100644 index 00000000000..d049f87578a --- /dev/null +++ b/db/post_migrate/20170526190948_create_index_in_pipeline_stages.rb @@ -0,0 +1,15 @@ +class CreateIndexInPipelineStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index(:ci_stages, [:pipeline_id, :name]) + end + + def down + remove_index(:ci_stages, [:pipeline_id, :name]) + end +end diff --git a/db/schema.rb b/db/schema.rb index f3bd9c26fb4..8d57159d2e0 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170526190708) do +ActiveRecord::Schema.define(version: 20170526190948) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" diff --git a/spec/migrations/migrate_pipeline_stages_spec.rb b/spec/migrations/migrate_pipeline_stages_spec.rb index 6887fcd26c3..c9b38086deb 100644 --- a/spec/migrations/migrate_pipeline_stages_spec.rb +++ b/spec/migrations/migrate_pipeline_stages_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' -require Rails.root.join('db', 'post_migrate', '20170526101042_migrate_pipeline_stages.rb') +require Rails.root.join('db', 'post_migrate', '20170526185842_migrate_pipeline_stages.rb') -describe MigratePipelineStages, :migration, schema: 20170525132202 do +describe MigratePipelineStages, :migration, schema: 20170526185602 do ## # Create test data - pipeline and CI/CD jobs. # diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index b4ece5c60c1..6a0e29f2edb 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -95,12 +95,14 @@ RSpec.configure do |config| end config.around(:example, migration: true) do |example| - schema_version = example.metadata.fetch(:schema) - ActiveRecord::Migrator.migrate(migrations_paths, schema_version) + begin + schema_version = example.metadata.fetch(:schema) + ActiveRecord::Migrator.migrate(migrations_paths, schema_version) - example.run - - ActiveRecord::Migrator.migrate(migrations_paths) + example.run + ensure + ActiveRecord::Migrator.migrate(migrations_paths) + end end end -- cgit v1.2.1 From 1a438e9f710edfa50408d93a363a240bdcd532c1 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 1 Jun 2017 12:21:34 +0200 Subject: Use stage_id column in code that needs to know about it --- spec/lib/gitlab/import_export/safe_model_attributes.yml | 1 + spec/services/ci/retry_build_service_spec.rb | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 96054c996fd..2388aea24d9 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -212,6 +212,7 @@ CommitStatus: - stage - trigger_request_id - stage_idx +- stage_id - tag - ref - user_id diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb index 7254e6b357a..2bd5af25847 100644 --- a/spec/services/ci/retry_build_service_spec.rb +++ b/spec/services/ci/retry_build_service_spec.rb @@ -18,11 +18,12 @@ describe Ci::RetryBuildService, :services do updated_at started_at finished_at queued_at erased_by erased_at auto_canceled_by].freeze + # TODO, move stage_id accessor to CLONE_ACCESSOR in a follow-up MR. IGNORE_ACCESSORS = %i[type lock_version target_url base_tags commit_id deployments erased_by_id last_deployment project_id runner_id tag_taggings taggings tags trigger_request_id - user_id auto_canceled_by_id retried].freeze + user_id auto_canceled_by_id retried stage_id].freeze shared_examples 'build duplication' do let(:build) do -- cgit v1.2.1 From 028423c2f51ff738d151df32254913664ac8e898 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 5 Jun 2017 11:07:10 +0200 Subject: Calculate previous migration version in specs support This makes it possible to test migration on the schema this migration was written for, without a need to specify a previous schema version manually. --- spec/migrations/migrate_build_stage_reference_spec.rb | 2 +- spec/migrations/migrate_pipeline_stages_spec.rb | 2 +- spec/spec_helper.rb | 6 +++--- spec/support/migrations_helpers.rb | 10 ++++++++++ 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/spec/migrations/migrate_build_stage_reference_spec.rb b/spec/migrations/migrate_build_stage_reference_spec.rb index 979f13a1398..eaac8f95892 100644 --- a/spec/migrations/migrate_build_stage_reference_spec.rb +++ b/spec/migrations/migrate_build_stage_reference_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20170526185921_migrate_build_stage_reference.rb') -describe MigrateBuildStageReference, :migration, schema: 20170526185602 do +describe MigrateBuildStageReference, :migration do ## # Create test data - pipeline and CI/CD jobs. # diff --git a/spec/migrations/migrate_pipeline_stages_spec.rb b/spec/migrations/migrate_pipeline_stages_spec.rb index c9b38086deb..36d3bd13ac0 100644 --- a/spec/migrations/migrate_pipeline_stages_spec.rb +++ b/spec/migrations/migrate_pipeline_stages_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20170526185842_migrate_pipeline_stages.rb') -describe MigratePipelineStages, :migration, schema: 20170526185602 do +describe MigratePipelineStages, :migration do ## # Create test data - pipeline and CI/CD jobs. # diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 6a0e29f2edb..5cbb3dafcdf 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -94,10 +94,10 @@ RSpec.configure do |config| Sidekiq.redis(&:flushall) end - config.around(:example, migration: true) do |example| + config.around(:example, :migration) do |example| begin - schema_version = example.metadata.fetch(:schema) - ActiveRecord::Migrator.migrate(migrations_paths, schema_version) + ActiveRecord::Migrator + .migrate(migrations_paths, previous_migration.version) example.run ensure diff --git a/spec/support/migrations_helpers.rb b/spec/support/migrations_helpers.rb index ee17d1a40b7..91fbb4eaf48 100644 --- a/spec/support/migrations_helpers.rb +++ b/spec/support/migrations_helpers.rb @@ -11,6 +11,16 @@ module MigrationsHelpers ActiveRecord::Base.connection.table_exists?(name) end + def migrations + ActiveRecord::Migrator.migrations(migrations_paths) + end + + def previous_migration + migrations.each_cons(2) do |previous, migration| + break previous if migration.name == described_class.name + end + end + def migrate! ActiveRecord::Migrator.up(migrations_paths) do |migration| migration.name == described_class.name -- cgit v1.2.1 From af72fa9c9c0af25cb7d8d349abd9dea4897aeea8 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 5 Jun 2017 13:21:49 +0200 Subject: Migrate pipeline stages only when not migrated already --- db/post_migrate/20170526185842_migrate_pipeline_stages.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/post_migrate/20170526185842_migrate_pipeline_stages.rb b/db/post_migrate/20170526185842_migrate_pipeline_stages.rb index 05e095f07cb..382791c4664 100644 --- a/db/post_migrate/20170526185842_migrate_pipeline_stages.rb +++ b/db/post_migrate/20170526185842_migrate_pipeline_stages.rb @@ -11,9 +11,9 @@ class MigratePipelineStages < ActiveRecord::Migration execute <<-SQL.strip_heredoc INSERT INTO ci_stages (project_id, pipeline_id, name) SELECT project_id, commit_id, stage FROM ci_builds - WHERE stage IS NOT NULL - GROUP BY project_id, commit_id, stage, stage_idx - ORDER BY stage_idx + WHERE stage IS NOT NULL AND stage_id IS NULL + GROUP BY project_id, commit_id, stage + ORDER BY MAX(stage_idx) SQL end -- cgit v1.2.1 From b72abd1d359da313b31097fcf042d2379e1c4ab2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 5 Jun 2017 13:24:07 +0200 Subject: Migrate stage_id only it job does not have it already --- db/post_migrate/20170526185921_migrate_build_stage_reference.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db/post_migrate/20170526185921_migrate_build_stage_reference.rb b/db/post_migrate/20170526185921_migrate_build_stage_reference.rb index 8f453b8cd82..3c30dfd6dde 100644 --- a/db/post_migrate/20170526185921_migrate_build_stage_reference.rb +++ b/db/post_migrate/20170526185921_migrate_build_stage_reference.rb @@ -10,7 +10,9 @@ class MigrateBuildStageReference < ActiveRecord::Migration 'WHERE ci_stages.pipeline_id = ci_builds.commit_id ' \ 'AND ci_stages.name = ci_builds.stage)') - update_column_in_batches(:ci_builds, :stage_id, stage_id) + update_column_in_batches(:ci_builds, :stage_id, stage_id) do |table, query| + query.where(table[:stage_id].eq(nil)) + end end def down -- cgit v1.2.1 From 06898af38f46daaa1c75cb4adead971062684875 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 5 Jun 2017 13:35:35 +0200 Subject: Create indexes on pipeline stages before migration Creates an index in ci_stages before migrating pipeline stages from ci_builds, to improve migration performance. --- .../20170526185748_create_index_in_pipeline_stages.rb | 15 +++++++++++++++ .../20170526190948_create_index_in_pipeline_stages.rb | 15 --------------- 2 files changed, 15 insertions(+), 15 deletions(-) create mode 100644 db/post_migrate/20170526185748_create_index_in_pipeline_stages.rb delete mode 100644 db/post_migrate/20170526190948_create_index_in_pipeline_stages.rb diff --git a/db/post_migrate/20170526185748_create_index_in_pipeline_stages.rb b/db/post_migrate/20170526185748_create_index_in_pipeline_stages.rb new file mode 100644 index 00000000000..d049f87578a --- /dev/null +++ b/db/post_migrate/20170526185748_create_index_in_pipeline_stages.rb @@ -0,0 +1,15 @@ +class CreateIndexInPipelineStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index(:ci_stages, [:pipeline_id, :name]) + end + + def down + remove_index(:ci_stages, [:pipeline_id, :name]) + end +end diff --git a/db/post_migrate/20170526190948_create_index_in_pipeline_stages.rb b/db/post_migrate/20170526190948_create_index_in_pipeline_stages.rb deleted file mode 100644 index d049f87578a..00000000000 --- a/db/post_migrate/20170526190948_create_index_in_pipeline_stages.rb +++ /dev/null @@ -1,15 +0,0 @@ -class CreateIndexInPipelineStages < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - disable_ddl_transaction! - - def up - add_concurrent_index(:ci_stages, [:pipeline_id, :name]) - end - - def down - remove_index(:ci_stages, [:pipeline_id, :name]) - end -end -- cgit v1.2.1 From 3326291f7e6c6b4452ca6aa4fc65faf1bd821220 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 5 Jun 2017 13:46:42 +0200 Subject: Remove obsolete stages/build before adding foreign keys --- ...0526190708_create_foreign_keys_for_pipeline_stages.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb b/db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb index 40060d12d74..8a01de1fce6 100644 --- a/db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb +++ b/db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb @@ -6,6 +6,22 @@ class CreateForeignKeysForPipelineStages < ActiveRecord::Migration disable_ddl_transaction! def up + execute <<~SQL + DELETE FROM ci_stages + WHERE NOT EXISTS ( + SELECT true FROM projects + WHERE projects.id = ci_stages.project_id + ) + SQL + + execute <<~SQL + DELETE FROM ci_builds + WHERE NOT EXISTS ( + SELECT true FROM ci_stages + WHERE ci_stages.id = ci_builds.stage_id + ) + SQL + add_concurrent_foreign_key :ci_stages, :projects, column: :project_id, on_delete: :cascade add_concurrent_foreign_key :ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade end -- cgit v1.2.1 From 7a2ad70f0b143e3fd195e6e5c475a7b7350c527c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 5 Jun 2017 13:48:41 +0200 Subject: Disable timeouts in foreign keys for stages migration --- .../20170526190708_create_foreign_keys_for_pipeline_stages.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb b/db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb index 8a01de1fce6..5ae3c1ae098 100644 --- a/db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb +++ b/db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb @@ -6,6 +6,8 @@ class CreateForeignKeysForPipelineStages < ActiveRecord::Migration disable_ddl_transaction! def up + disable_statement_timeout + execute <<~SQL DELETE FROM ci_stages WHERE NOT EXISTS ( -- cgit v1.2.1 From faeb2f3a09d18a0041a6c425a60f26ed9a41498a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 5 Jun 2017 14:54:39 +0200 Subject: Remove stage index concurrently on migration rollback --- db/post_migrate/20170526185748_create_index_in_pipeline_stages.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/post_migrate/20170526185748_create_index_in_pipeline_stages.rb b/db/post_migrate/20170526185748_create_index_in_pipeline_stages.rb index d049f87578a..ec9ff33b6b7 100644 --- a/db/post_migrate/20170526185748_create_index_in_pipeline_stages.rb +++ b/db/post_migrate/20170526185748_create_index_in_pipeline_stages.rb @@ -10,6 +10,6 @@ class CreateIndexInPipelineStages < ActiveRecord::Migration end def down - remove_index(:ci_stages, [:pipeline_id, :name]) + remove_concurrent_index(:ci_stages, [:pipeline_id, :name]) end end -- cgit v1.2.1 From cafb1bfea4dbc7b6aad47580e76d68b8075d177f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 5 Jun 2017 14:56:53 +0200 Subject: Use the latest migration version as a schema version --- db/schema.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index 47484e6b8fe..23c3ecdd517 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170526190948) do +ActiveRecord::Schema.define(version: 20170526190708) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" -- cgit v1.2.1 From 5a31bbe86ca41b1461ed8bf5a663aa51c123f149 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 6 Jun 2017 13:26:19 +0200 Subject: Make pipeline stages ref migration more readable --- db/post_migrate/20170526185921_migrate_build_stage_reference.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/db/post_migrate/20170526185921_migrate_build_stage_reference.rb b/db/post_migrate/20170526185921_migrate_build_stage_reference.rb index 3c30dfd6dde..aa50b72839d 100644 --- a/db/post_migrate/20170526185921_migrate_build_stage_reference.rb +++ b/db/post_migrate/20170526185921_migrate_build_stage_reference.rb @@ -6,9 +6,11 @@ class MigrateBuildStageReference < ActiveRecord::Migration def up disable_statement_timeout - stage_id = Arel.sql('(SELECT id FROM ci_stages ' \ - 'WHERE ci_stages.pipeline_id = ci_builds.commit_id ' \ - 'AND ci_stages.name = ci_builds.stage)') + stage_id = Arel.sql(<<-SQL.strip_heredoc + (SELECT id FROM ci_stages + WHERE ci_stages.pipeline_id = ci_builds.commit_id + AND ci_stages.name = ci_builds.stage) + SQL update_column_in_batches(:ci_builds, :stage_id, stage_id) do |table, query| query.where(table[:stage_id].eq(nil)) -- cgit v1.2.1 From d56460682061b38557ef107b72358c76dee7509f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 6 Jun 2017 13:30:29 +0200 Subject: Improve order of columns in pipeline stages table --- db/migrate/20170525132202_create_pipeline_stages.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/migrate/20170525132202_create_pipeline_stages.rb b/db/migrate/20170525132202_create_pipeline_stages.rb index c562df27357..634f2c1156e 100644 --- a/db/migrate/20170525132202_create_pipeline_stages.rb +++ b/db/migrate/20170525132202_create_pipeline_stages.rb @@ -7,8 +7,8 @@ class CreatePipelineStages < ActiveRecord::Migration create_table :ci_stages do |t| t.integer :project_id t.integer :pipeline_id - t.string :name t.timestamps null: true + t.string :name end end end -- cgit v1.2.1 From cc1e2a2bef324fe5869393e04be2feed22576667 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 6 Jun 2017 13:36:56 +0200 Subject: Fix typo in build stages reference migration --- db/post_migrate/20170526185921_migrate_build_stage_reference.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/post_migrate/20170526185921_migrate_build_stage_reference.rb b/db/post_migrate/20170526185921_migrate_build_stage_reference.rb index aa50b72839d..797e106cae4 100644 --- a/db/post_migrate/20170526185921_migrate_build_stage_reference.rb +++ b/db/post_migrate/20170526185921_migrate_build_stage_reference.rb @@ -6,7 +6,7 @@ class MigrateBuildStageReference < ActiveRecord::Migration def up disable_statement_timeout - stage_id = Arel.sql(<<-SQL.strip_heredoc + stage_id = Arel.sql <<-SQL.strip_heredoc (SELECT id FROM ci_stages WHERE ci_stages.pipeline_id = ci_builds.commit_id AND ci_stages.name = ci_builds.stage) -- cgit v1.2.1 From 8e071f28097b2c55b9e8d1b3a7ced4cdd04ef44d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 6 Jun 2017 14:10:50 +0200 Subject: Improve indexes and refs in pipeline stages migrations --- .../20170525132202_create_pipeline_stages.rb | 13 +++++++- .../20170526185602_add_stage_id_to_ci_builds.rb | 14 ++++++++- .../20170526185842_migrate_pipeline_stages.rb | 4 --- ...0708_create_foreign_keys_for_pipeline_stages.rb | 35 ---------------------- db/schema.rb | 8 +++-- 5 files changed, 31 insertions(+), 43 deletions(-) delete mode 100644 db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb diff --git a/db/migrate/20170525132202_create_pipeline_stages.rb b/db/migrate/20170525132202_create_pipeline_stages.rb index 634f2c1156e..25656f2a2c2 100644 --- a/db/migrate/20170525132202_create_pipeline_stages.rb +++ b/db/migrate/20170525132202_create_pipeline_stages.rb @@ -3,12 +3,23 @@ class CreatePipelineStages < ActiveRecord::Migration DOWNTIME = false - def change + disable_ddl_transaction! + + def up create_table :ci_stages do |t| t.integer :project_id t.integer :pipeline_id t.timestamps null: true t.string :name end + + add_concurrent_foreign_key :ci_stages, :projects, column: :project_id, on_delete: :cascade + add_concurrent_foreign_key :ci_stages, :ci_pipelines, column: :pipeline_id, on_delete: :cascade + add_concurrent_index :ci_stages, :project_id + add_concurrent_index :ci_stages, :pipeline_id + end + + def down + drop_table :ci_stages end end diff --git a/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb b/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb index 0788f15f7c5..e8009e74ea5 100644 --- a/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb +++ b/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb @@ -3,7 +3,19 @@ class AddStageIdToCiBuilds < ActiveRecord::Migration DOWNTIME = false - def change + disable_ddl_transaction! + + def up add_column :ci_builds, :stage_id, :integer + + add_concurrent_foreign_key :ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade + add_concurrent_index :ci_builds, :stage_id + end + + def down + remove_concurrent_index :ci_builds, :stage_id + remove_foreign_key :ci_builds, column: :stage_id + + remove_column :ci_builds, :stage_id, :integer end end diff --git a/db/post_migrate/20170526185842_migrate_pipeline_stages.rb b/db/post_migrate/20170526185842_migrate_pipeline_stages.rb index 382791c4664..37ccf9fc105 100644 --- a/db/post_migrate/20170526185842_migrate_pipeline_stages.rb +++ b/db/post_migrate/20170526185842_migrate_pipeline_stages.rb @@ -16,8 +16,4 @@ class MigratePipelineStages < ActiveRecord::Migration ORDER BY MAX(stage_idx) SQL end - - def down - execute('TRUNCATE TABLE ci_stages') - end end diff --git a/db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb b/db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb deleted file mode 100644 index 5ae3c1ae098..00000000000 --- a/db/post_migrate/20170526190708_create_foreign_keys_for_pipeline_stages.rb +++ /dev/null @@ -1,35 +0,0 @@ -class CreateForeignKeysForPipelineStages < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - disable_ddl_transaction! - - def up - disable_statement_timeout - - execute <<~SQL - DELETE FROM ci_stages - WHERE NOT EXISTS ( - SELECT true FROM projects - WHERE projects.id = ci_stages.project_id - ) - SQL - - execute <<~SQL - DELETE FROM ci_builds - WHERE NOT EXISTS ( - SELECT true FROM ci_stages - WHERE ci_stages.id = ci_builds.stage_id - ) - SQL - - add_concurrent_foreign_key :ci_stages, :projects, column: :project_id, on_delete: :cascade - add_concurrent_foreign_key :ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade - end - - def down - remove_foreign_key :ci_stages, column: :project_id - remove_foreign_key :ci_builds, column: :stage_id - end -end diff --git a/db/schema.rb b/db/schema.rb index 23c3ecdd517..06dfea8cfb5 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170526190708) do +ActiveRecord::Schema.define(version: 20170526185921) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -243,6 +243,7 @@ ActiveRecord::Schema.define(version: 20170526190708) do add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree add_index "ci_builds", ["project_id"], name: "index_ci_builds_on_project_id", using: :btree add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree + add_index "ci_builds", ["stage_id"], name: "index_ci_builds_on_stage_id", using: :btree add_index "ci_builds", ["status", "type", "runner_id"], name: "index_ci_builds_on_status_and_type_and_runner_id", using: :btree add_index "ci_builds", ["status"], name: "index_ci_builds_on_status", using: :btree add_index "ci_builds", ["token"], name: "index_ci_builds_on_token", unique: true, using: :btree @@ -330,12 +331,14 @@ ActiveRecord::Schema.define(version: 20170526190708) do create_table "ci_stages", force: :cascade do |t| t.integer "project_id" t.integer "pipeline_id" - t.string "name" t.datetime "created_at" t.datetime "updated_at" + t.string "name" end add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", using: :btree + add_index "ci_stages", ["pipeline_id"], name: "index_ci_stages_on_pipeline_id", using: :btree + add_index "ci_stages", ["project_id"], name: "index_ci_stages_on_project_id", using: :btree create_table "ci_trigger_requests", force: :cascade do |t| t.integer "trigger_id", null: false @@ -1497,6 +1500,7 @@ ActiveRecord::Schema.define(version: 20170526190708) do add_foreign_key "ci_pipeline_schedules", "users", column: "owner_id", name: "fk_9ea99f58d2", on_delete: :nullify add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify + add_foreign_key "ci_stages", "ci_pipelines", column: "pipeline_id", name: "fk_fb57e6cc56", on_delete: :cascade add_foreign_key "ci_stages", "projects", name: "fk_2360681d1d", on_delete: :cascade add_foreign_key "ci_trigger_requests", "ci_triggers", column: "trigger_id", name: "fk_b8ec8b7245", on_delete: :cascade add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade -- cgit v1.2.1 From 559521ce484d4a95bf3b6ac23bd48c497fdf2703 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 6 Jun 2017 14:13:51 +0200 Subject: Create index on pipeline stages after migrating stages --- .../20170526185748_create_index_in_pipeline_stages.rb | 15 --------------- .../20170526185858_create_index_in_pipeline_stages.rb | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 15 deletions(-) delete mode 100644 db/post_migrate/20170526185748_create_index_in_pipeline_stages.rb create mode 100644 db/post_migrate/20170526185858_create_index_in_pipeline_stages.rb diff --git a/db/post_migrate/20170526185748_create_index_in_pipeline_stages.rb b/db/post_migrate/20170526185748_create_index_in_pipeline_stages.rb deleted file mode 100644 index ec9ff33b6b7..00000000000 --- a/db/post_migrate/20170526185748_create_index_in_pipeline_stages.rb +++ /dev/null @@ -1,15 +0,0 @@ -class CreateIndexInPipelineStages < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - disable_ddl_transaction! - - def up - add_concurrent_index(:ci_stages, [:pipeline_id, :name]) - end - - def down - remove_concurrent_index(:ci_stages, [:pipeline_id, :name]) - end -end diff --git a/db/post_migrate/20170526185858_create_index_in_pipeline_stages.rb b/db/post_migrate/20170526185858_create_index_in_pipeline_stages.rb new file mode 100644 index 00000000000..ec9ff33b6b7 --- /dev/null +++ b/db/post_migrate/20170526185858_create_index_in_pipeline_stages.rb @@ -0,0 +1,15 @@ +class CreateIndexInPipelineStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index(:ci_stages, [:pipeline_id, :name]) + end + + def down + remove_concurrent_index(:ci_stages, [:pipeline_id, :name]) + end +end -- cgit v1.2.1 From b55aad4cc3bf7723f4531635bbb31ab44cae17a9 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 6 Jun 2017 14:32:24 +0200 Subject: Migrate stages only with correct foreign references --- db/post_migrate/20170526185842_migrate_pipeline_stages.rb | 5 ++++- spec/migrations/migrate_build_stage_reference_spec.rb | 7 +++++++ spec/migrations/migrate_pipeline_stages_spec.rb | 15 ++++++++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/db/post_migrate/20170526185842_migrate_pipeline_stages.rb b/db/post_migrate/20170526185842_migrate_pipeline_stages.rb index 37ccf9fc105..afd4db183c2 100644 --- a/db/post_migrate/20170526185842_migrate_pipeline_stages.rb +++ b/db/post_migrate/20170526185842_migrate_pipeline_stages.rb @@ -11,7 +11,10 @@ class MigratePipelineStages < ActiveRecord::Migration execute <<-SQL.strip_heredoc INSERT INTO ci_stages (project_id, pipeline_id, name) SELECT project_id, commit_id, stage FROM ci_builds - WHERE stage IS NOT NULL AND stage_id IS NULL + WHERE stage IS NOT NULL + AND stage_id IS NULL + AND EXISTS (SELECT 1 FROM projects WHERE projects.id = ci_builds.project_id) + AND EXISTS (SELECT 1 FROM ci_pipelines WHERE ci_pipelines.id = ci_builds.commit_id) GROUP BY project_id, commit_id, stage ORDER BY MAX(stage_idx) SQL diff --git a/spec/migrations/migrate_build_stage_reference_spec.rb b/spec/migrations/migrate_build_stage_reference_spec.rb index eaac8f95892..32f9a0c0642 100644 --- a/spec/migrations/migrate_build_stage_reference_spec.rb +++ b/spec/migrations/migrate_build_stage_reference_spec.rb @@ -9,8 +9,15 @@ describe MigrateBuildStageReference, :migration do let(:jobs) { table(:ci_builds) } let(:stages) { table(:ci_stages) } let(:pipelines) { table(:ci_pipelines) } + let(:projects) { table(:projects) } before do + # Create projects + # + projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1') + projects.create!(id: 456, name: 'gitlab2', path: 'gitlab2') + projects.create!(id: 798, name: 'gitlab3', path: 'gitlab3') + # Create CI/CD pipelines # pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') diff --git a/spec/migrations/migrate_pipeline_stages_spec.rb b/spec/migrations/migrate_pipeline_stages_spec.rb index 36d3bd13ac0..c47f2bb8ff9 100644 --- a/spec/migrations/migrate_pipeline_stages_spec.rb +++ b/spec/migrations/migrate_pipeline_stages_spec.rb @@ -9,8 +9,14 @@ describe MigratePipelineStages, :migration do let(:jobs) { table(:ci_builds) } let(:stages) { table(:ci_stages) } let(:pipelines) { table(:ci_pipelines) } + let(:projects) { table(:projects) } before do + # Create projects + # + projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1') + projects.create!(id: 456, name: 'gitlab2', path: 'gitlab2') + # Create CI/CD pipelines # pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') @@ -28,7 +34,8 @@ describe MigratePipelineStages, :migration do jobs.create!(id: 8, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') jobs.create!(id: 9, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') jobs.create!(id: 10, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2') - jobs.create!(id: 11, commit_id: 3, project_id: 789, stage_idx: 3, stage: 'deploy') + jobs.create!(id: 11, commit_id: 3, project_id: 456, stage_idx: 3, stage: 'deploy') + jobs.create!(id: 12, commit_id: 2, project_id: 789, stage_idx: 3, stage: 'deploy') end it 'correctly migrates pipeline stages' do @@ -36,12 +43,14 @@ describe MigratePipelineStages, :migration do migrate! - expect(stages.count).to eq 7 + expect(stages.count).to eq 6 expect(stages.all.pluck(:name)) - .to match_array %w[test build deploy test:1 test:2 deploy deploy] + .to match_array %w[test build deploy test:1 test:2 deploy] expect(stages.where(pipeline_id: 1).order(:id).pluck(:name)) .to eq %w[test build deploy] expect(stages.where(pipeline_id: 2).order(:id).pluck(:name)) .to eq %w[test:1 test:2 deploy] + expect(stages.where(pipeline_id: 3).count).to be_zero + expect(stages.where(project_id: 789).count).to be_zero end end -- cgit v1.2.1 From da8bd81fe7c1042c50d7c693fdf37ea5b7b78285 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 6 Jun 2017 16:45:24 +0200 Subject: Fix reverting builds stage ref migration on MySQL --- db/migrate/20170526185602_add_stage_id_to_ci_builds.rb | 2 +- spec/migrations/migrate_build_stage_reference_spec.rb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb b/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb index e8009e74ea5..d5675d5828b 100644 --- a/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb +++ b/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb @@ -13,8 +13,8 @@ class AddStageIdToCiBuilds < ActiveRecord::Migration end def down - remove_concurrent_index :ci_builds, :stage_id remove_foreign_key :ci_builds, column: :stage_id + remove_concurrent_index :ci_builds, :stage_id remove_column :ci_builds, :stage_id, :integer end diff --git a/spec/migrations/migrate_build_stage_reference_spec.rb b/spec/migrations/migrate_build_stage_reference_spec.rb index 32f9a0c0642..80b321860c2 100644 --- a/spec/migrations/migrate_build_stage_reference_spec.rb +++ b/spec/migrations/migrate_build_stage_reference_spec.rb @@ -16,7 +16,6 @@ describe MigrateBuildStageReference, :migration do # projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1') projects.create!(id: 456, name: 'gitlab2', path: 'gitlab2') - projects.create!(id: 798, name: 'gitlab3', path: 'gitlab3') # Create CI/CD pipelines # -- cgit v1.2.1