From 6b9608e13e20fe579b861b911880603a372f2f27 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 21 Jun 2017 10:32:45 +0200 Subject: Fix build stages reference migration --- .../20170526185602_add_stage_id_to_ci_builds.rb | 4 ---- ...0170526185901_remove_stage_id_index_from_builds.rb | 17 +++++++++++++++++ .../20170526185921_migrate_build_stage_reference.rb | 2 ++ .../20170621102400_add_stage_id_index_to_builds.rb | 19 +++++++++++++++++++ db/schema.rb | 2 +- 5 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 db/post_migrate/20170526185901_remove_stage_id_index_from_builds.rb create mode 100644 db/post_migrate/20170621102400_add_stage_id_index_to_builds.rb diff --git a/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb b/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb index d5675d5828b..2eeb2c481eb 100644 --- a/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb +++ b/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb @@ -7,15 +7,11 @@ class AddStageIdToCiBuilds < ActiveRecord::Migration def up add_column :ci_builds, :stage_id, :integer - add_concurrent_foreign_key :ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade - add_concurrent_index :ci_builds, :stage_id end def down remove_foreign_key :ci_builds, column: :stage_id - remove_concurrent_index :ci_builds, :stage_id - remove_column :ci_builds, :stage_id, :integer end end diff --git a/db/post_migrate/20170526185901_remove_stage_id_index_from_builds.rb b/db/post_migrate/20170526185901_remove_stage_id_index_from_builds.rb new file mode 100644 index 00000000000..a2c0b0c651b --- /dev/null +++ b/db/post_migrate/20170526185901_remove_stage_id_index_from_builds.rb @@ -0,0 +1,17 @@ +class RemoveStageIdIndexFromBuilds < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + if index_exists?(:ci_builds, :stage_id) + remove_concurrent_index(:ci_builds, :stage_id) + end + end + + def down + # noop + end +end diff --git a/db/post_migrate/20170526185921_migrate_build_stage_reference.rb b/db/post_migrate/20170526185921_migrate_build_stage_reference.rb index 797e106cae4..d2e7fc207f0 100644 --- a/db/post_migrate/20170526185921_migrate_build_stage_reference.rb +++ b/db/post_migrate/20170526185921_migrate_build_stage_reference.rb @@ -3,6 +3,8 @@ class MigrateBuildStageReference < ActiveRecord::Migration DOWNTIME = false + disable_ddl_transaction! + def up disable_statement_timeout diff --git a/db/post_migrate/20170621102400_add_stage_id_index_to_builds.rb b/db/post_migrate/20170621102400_add_stage_id_index_to_builds.rb new file mode 100644 index 00000000000..acbe16812d2 --- /dev/null +++ b/db/post_migrate/20170621102400_add_stage_id_index_to_builds.rb @@ -0,0 +1,19 @@ +class AddStageIdIndexToBuilds < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + unless index_exists?(:ci_builds, :stage_id) + add_concurrent_index(:ci_builds, :stage_id) + end + end + + def down + if index_exists?(:ci_builds, :stage_id) + remove_concurrent_index(:ci_builds, :stage_id) + end + end +end diff --git a/db/schema.rb b/db/schema.rb index f42827991aa..ec1329892eb 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170614115405) do +ActiveRecord::Schema.define(version: 20170621102400) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" -- cgit v1.2.1 From 885d63d9ed5802803fa4797e63f4f7e02c5a73ab Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 21 Jun 2017 11:02:18 +0200 Subject: Fix adding/removing foreign keys on MySQL --- db/migrate/20170526185602_add_stage_id_to_ci_builds.rb | 4 ---- db/post_migrate/20170526185901_remove_stage_id_index_from_builds.rb | 1 + db/post_migrate/20170621102400_add_stage_id_index_to_builds.rb | 2 ++ 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb b/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb index 2eeb2c481eb..d27cba76d81 100644 --- a/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb +++ b/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb @@ -3,15 +3,11 @@ class AddStageIdToCiBuilds < ActiveRecord::Migration DOWNTIME = false - disable_ddl_transaction! - def up add_column :ci_builds, :stage_id, :integer - add_concurrent_foreign_key :ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade end def down - remove_foreign_key :ci_builds, column: :stage_id remove_column :ci_builds, :stage_id, :integer end end diff --git a/db/post_migrate/20170526185901_remove_stage_id_index_from_builds.rb b/db/post_migrate/20170526185901_remove_stage_id_index_from_builds.rb index a2c0b0c651b..3879cf9133b 100644 --- a/db/post_migrate/20170526185901_remove_stage_id_index_from_builds.rb +++ b/db/post_migrate/20170526185901_remove_stage_id_index_from_builds.rb @@ -7,6 +7,7 @@ class RemoveStageIdIndexFromBuilds < ActiveRecord::Migration def up if index_exists?(:ci_builds, :stage_id) + remove_foreign_key(:ci_builds, column: :stage_id) remove_concurrent_index(:ci_builds, :stage_id) end end diff --git a/db/post_migrate/20170621102400_add_stage_id_index_to_builds.rb b/db/post_migrate/20170621102400_add_stage_id_index_to_builds.rb index acbe16812d2..7d6609b18bf 100644 --- a/db/post_migrate/20170621102400_add_stage_id_index_to_builds.rb +++ b/db/post_migrate/20170621102400_add_stage_id_index_to_builds.rb @@ -7,12 +7,14 @@ class AddStageIdIndexToBuilds < ActiveRecord::Migration def up unless index_exists?(:ci_builds, :stage_id) + add_concurrent_foreign_key(:ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade) add_concurrent_index(:ci_builds, :stage_id) end end def down if index_exists?(:ci_builds, :stage_id) + remove_foreign_key(:ci_builds, column: :stage_id) remove_concurrent_index(:ci_builds, :stage_id) end end -- cgit v1.2.1 From 090ed087f0aba10dff142e48d97259fec28b89bb Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 21 Jun 2017 13:55:28 +0200 Subject: Bump build stage id reference migration identifier --- ...20170526185921_migrate_build_stage_reference.rb | 22 ++++++------------ ...26190000_migrate_build_stage_reference_again.rb | 27 ++++++++++++++++++++++ 2 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 db/post_migrate/20170526190000_migrate_build_stage_reference_again.rb diff --git a/db/post_migrate/20170526185921_migrate_build_stage_reference.rb b/db/post_migrate/20170526185921_migrate_build_stage_reference.rb index d2e7fc207f0..98c32d8284c 100644 --- a/db/post_migrate/20170526185921_migrate_build_stage_reference.rb +++ b/db/post_migrate/20170526185921_migrate_build_stage_reference.rb @@ -3,25 +3,17 @@ class MigrateBuildStageReference < ActiveRecord::Migration DOWNTIME = false - disable_ddl_transaction! + ## + # This is an empty migration, content has been moved to a new one: + # post migrate 20170526190000 MigrateBuildStageReferenceAgain + # + # See gitlab-org/gitlab-ce!12337 for more details. def up - disable_statement_timeout - - stage_id = Arel.sql <<-SQL.strip_heredoc - (SELECT id FROM ci_stages - WHERE ci_stages.pipeline_id = ci_builds.commit_id - AND ci_stages.name = ci_builds.stage) - SQL - - update_column_in_batches(:ci_builds, :stage_id, stage_id) do |table, query| - query.where(table[:stage_id].eq(nil)) - end + # noop end def down - disable_statement_timeout - - update_column_in_batches(:ci_builds, :stage_id, nil) + # noop end end diff --git a/db/post_migrate/20170526190000_migrate_build_stage_reference_again.rb b/db/post_migrate/20170526190000_migrate_build_stage_reference_again.rb new file mode 100644 index 00000000000..97cb242415d --- /dev/null +++ b/db/post_migrate/20170526190000_migrate_build_stage_reference_again.rb @@ -0,0 +1,27 @@ +class MigrateBuildStageReferenceAgain < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + disable_statement_timeout + + stage_id = Arel.sql <<-SQL.strip_heredoc + (SELECT id FROM ci_stages + WHERE ci_stages.pipeline_id = ci_builds.commit_id + AND ci_stages.name = ci_builds.stage) + SQL + + update_column_in_batches(:ci_builds, :stage_id, stage_id) do |table, query| + query.where(table[:stage_id].eq(nil)) + end + end + + def down + disable_statement_timeout + + update_column_in_batches(:ci_builds, :stage_id, nil) + end +end -- cgit v1.2.1 From 2e3c1ae1f0404ab443e6f82a4b418e1322ea0c81 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 21 Jun 2017 14:42:53 +0200 Subject: Update build stage reference migration specs --- spec/migrations/migrate_build_stage_reference_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/migrations/migrate_build_stage_reference_spec.rb b/spec/migrations/migrate_build_stage_reference_spec.rb index 80b321860c2..6be480ce58e 100644 --- a/spec/migrations/migrate_build_stage_reference_spec.rb +++ b/spec/migrations/migrate_build_stage_reference_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' -require Rails.root.join('db', 'post_migrate', '20170526185921_migrate_build_stage_reference.rb') +require Rails.root.join('db', 'post_migrate', '20170526190000_migrate_build_stage_reference_again.rb') -describe MigrateBuildStageReference, :migration do +describe MigrateBuildStageReferenceAgain, :migration do ## # Create test data - pipeline and CI/CD jobs. # -- cgit v1.2.1 From 1dce0836d47395787e6764ab5a20ee1073cc7e32 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 21 Jun 2017 15:51:00 +0200 Subject: Rename stage ref migration specs to match a class name --- .../migrate_build_stage_reference_again_spec.rb | 62 ++++++++++++++++++++++ .../migrate_build_stage_reference_spec.rb | 62 ---------------------- 2 files changed, 62 insertions(+), 62 deletions(-) create mode 100644 spec/migrations/migrate_build_stage_reference_again_spec.rb delete mode 100644 spec/migrations/migrate_build_stage_reference_spec.rb diff --git a/spec/migrations/migrate_build_stage_reference_again_spec.rb b/spec/migrations/migrate_build_stage_reference_again_spec.rb new file mode 100644 index 00000000000..6be480ce58e --- /dev/null +++ b/spec/migrations/migrate_build_stage_reference_again_spec.rb @@ -0,0 +1,62 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20170526190000_migrate_build_stage_reference_again.rb') + +describe MigrateBuildStageReferenceAgain, :migration do + ## + # Create test data - pipeline and CI/CD jobs. + # + + let(:jobs) { table(:ci_builds) } + let(:stages) { table(:ci_stages) } + let(:pipelines) { table(:ci_pipelines) } + let(:projects) { table(:projects) } + + before do + # Create projects + # + projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1') + projects.create!(id: 456, name: 'gitlab2', path: 'gitlab2') + + # Create CI/CD pipelines + # + pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') + pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb') + + # Create CI/CD jobs + # + jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') + jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') + jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test') + jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy') + jobs.create!(id: 5, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2') + jobs.create!(id: 6, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') + jobs.create!(id: 7, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') + jobs.create!(id: 8, commit_id: 3, project_id: 789, stage_idx: 3, stage: 'deploy') + + # Create CI/CD stages + # + stages.create(id: 101, pipeline_id: 1, project_id: 123, name: 'test') + stages.create(id: 102, pipeline_id: 1, project_id: 123, name: 'build') + stages.create(id: 103, pipeline_id: 1, project_id: 123, name: 'deploy') + stages.create(id: 104, pipeline_id: 2, project_id: 456, name: 'test:1') + stages.create(id: 105, pipeline_id: 2, project_id: 456, name: 'test:2') + stages.create(id: 106, pipeline_id: 2, project_id: 456, name: 'deploy') + end + + it 'correctly migrate build stage references' do + expect(jobs.where(stage_id: nil).count).to eq 8 + + migrate! + + expect(jobs.where(stage_id: nil).count).to eq 1 + + expect(jobs.find(1).stage_id).to eq 102 + expect(jobs.find(2).stage_id).to eq 102 + expect(jobs.find(3).stage_id).to eq 101 + expect(jobs.find(4).stage_id).to eq 103 + expect(jobs.find(5).stage_id).to eq 105 + expect(jobs.find(6).stage_id).to eq 104 + expect(jobs.find(7).stage_id).to eq 104 + expect(jobs.find(8).stage_id).to eq nil + end +end diff --git a/spec/migrations/migrate_build_stage_reference_spec.rb b/spec/migrations/migrate_build_stage_reference_spec.rb deleted file mode 100644 index 6be480ce58e..00000000000 --- a/spec/migrations/migrate_build_stage_reference_spec.rb +++ /dev/null @@ -1,62 +0,0 @@ -require 'spec_helper' -require Rails.root.join('db', 'post_migrate', '20170526190000_migrate_build_stage_reference_again.rb') - -describe MigrateBuildStageReferenceAgain, :migration do - ## - # Create test data - pipeline and CI/CD jobs. - # - - let(:jobs) { table(:ci_builds) } - let(:stages) { table(:ci_stages) } - let(:pipelines) { table(:ci_pipelines) } - let(:projects) { table(:projects) } - - before do - # Create projects - # - projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1') - projects.create!(id: 456, name: 'gitlab2', path: 'gitlab2') - - # Create CI/CD pipelines - # - pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') - pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb') - - # Create CI/CD jobs - # - jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') - jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') - jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test') - jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy') - jobs.create!(id: 5, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2') - jobs.create!(id: 6, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') - jobs.create!(id: 7, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') - jobs.create!(id: 8, commit_id: 3, project_id: 789, stage_idx: 3, stage: 'deploy') - - # Create CI/CD stages - # - stages.create(id: 101, pipeline_id: 1, project_id: 123, name: 'test') - stages.create(id: 102, pipeline_id: 1, project_id: 123, name: 'build') - stages.create(id: 103, pipeline_id: 1, project_id: 123, name: 'deploy') - stages.create(id: 104, pipeline_id: 2, project_id: 456, name: 'test:1') - stages.create(id: 105, pipeline_id: 2, project_id: 456, name: 'test:2') - stages.create(id: 106, pipeline_id: 2, project_id: 456, name: 'deploy') - end - - it 'correctly migrate build stage references' do - expect(jobs.where(stage_id: nil).count).to eq 8 - - migrate! - - expect(jobs.where(stage_id: nil).count).to eq 1 - - expect(jobs.find(1).stage_id).to eq 102 - expect(jobs.find(2).stage_id).to eq 102 - expect(jobs.find(3).stage_id).to eq 101 - expect(jobs.find(4).stage_id).to eq 103 - expect(jobs.find(5).stage_id).to eq 105 - expect(jobs.find(6).stage_id).to eq 104 - expect(jobs.find(7).stage_id).to eq 104 - expect(jobs.find(8).stage_id).to eq nil - end -end -- cgit v1.2.1