Merge branch 'feature/gb/migrate-pipeline-stages' into 'master'

Migrate pipeline stages in the database

See merge request !11714

7 files changed, 166 insertions, 1 deletions

diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml
index 3b6bba9ffdd..de7852aa810 100644
--- a/spec/lib/gitlab/import_export/safe_model_attributes.yml
+++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml
@@ -214,6 +214,7 @@ CommitStatus:
 - stage
 - trigger_request_id
 - stage_idx
+- stage_id
 - tag
 - ref
 - user_id
diff --git a/spec/migrations/migrate_build_stage_reference_spec.rb b/spec/migrations/migrate_build_stage_reference_spec.rb
new file mode 100644
index 00000000000..80b321860c2
--- /dev/null
+++ b/spec/migrations/migrate_build_stage_reference_spec.rb
@@ -0,0 +1,62 @@
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20170526185921_migrate_build_stage_reference.rb')
+
+describe MigrateBuildStageReference, :migration do
+  ##
+  # Create test data - pipeline and CI/CD jobs.
+  #
+
+  let(:jobs) { table(:ci_builds) }
+  let(:stages) { table(:ci_stages) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:projects) { table(:projects) }
+
+  before do
+    # Create projects
+    #
+    projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
+    projects.create!(id: 456, name: 'gitlab2', path: 'gitlab2')
+
+    # Create CI/CD pipelines
+    #
+    pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
+    pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb')
+
+    # Create CI/CD jobs
+    #
+    jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
+    jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
+    jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test')
+    jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy')
+    jobs.create!(id: 5, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2')
+    jobs.create!(id: 6, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1')
+    jobs.create!(id: 7, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1')
+    jobs.create!(id: 8, commit_id: 3, project_id: 789, stage_idx: 3, stage: 'deploy')
+
+    # Create CI/CD stages
+    #
+    stages.create(id: 101, pipeline_id: 1, project_id: 123, name: 'test')
+    stages.create(id: 102, pipeline_id: 1, project_id: 123, name: 'build')
+    stages.create(id: 103, pipeline_id: 1, project_id: 123, name: 'deploy')
+    stages.create(id: 104, pipeline_id: 2, project_id: 456, name: 'test:1')
+    stages.create(id: 105, pipeline_id: 2, project_id: 456, name: 'test:2')
+    stages.create(id: 106, pipeline_id: 2, project_id: 456, name: 'deploy')
+  end
+
+  it 'correctly migrate build stage references' do
+    expect(jobs.where(stage_id: nil).count).to eq 8
+
+    migrate!
+
+    expect(jobs.where(stage_id: nil).count).to eq 1
+
+    expect(jobs.find(1).stage_id).to eq 102
+    expect(jobs.find(2).stage_id).to eq 102
+    expect(jobs.find(3).stage_id).to eq 101
+    expect(jobs.find(4).stage_id).to eq 103
+    expect(jobs.find(5).stage_id).to eq 105
+    expect(jobs.find(6).stage_id).to eq 104
+    expect(jobs.find(7).stage_id).to eq 104
+    expect(jobs.find(8).stage_id).to eq nil
+  end
+end
diff --git a/spec/migrations/migrate_pipeline_stages_spec.rb b/spec/migrations/migrate_pipeline_stages_spec.rb
new file mode 100644
index 00000000000..c47f2bb8ff9
--- /dev/null
+++ b/spec/migrations/migrate_pipeline_stages_spec.rb
@@ -0,0 +1,56 @@
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20170526185842_migrate_pipeline_stages.rb')
+
+describe MigratePipelineStages, :migration do
+  ##
+  # Create test data - pipeline and CI/CD jobs.
+  #
+
+  let(:jobs) { table(:ci_builds) }
+  let(:stages) { table(:ci_stages) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:projects) { table(:projects) }
+
+  before do
+    # Create projects
+    #
+    projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
+    projects.create!(id: 456, name: 'gitlab2', path: 'gitlab2')
+
+    # Create CI/CD pipelines
+    #
+    pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
+    pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb')
+
+    # Create CI/CD jobs
+    #
+    jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
+    jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
+    jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test')
+    jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test')
+    jobs.create!(id: 5, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy')
+    jobs.create!(id: 6, commit_id: 2, project_id: 456, stage_idx: 3, stage: 'deploy')
+    jobs.create!(id: 7, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2')
+    jobs.create!(id: 8, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1')
+    jobs.create!(id: 9, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1')
+    jobs.create!(id: 10, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2')
+    jobs.create!(id: 11, commit_id: 3, project_id: 456, stage_idx: 3, stage: 'deploy')
+    jobs.create!(id: 12, commit_id: 2, project_id: 789, stage_idx: 3, stage: 'deploy')
+  end
+
+  it 'correctly migrates pipeline stages' do
+    expect(stages.count).to be_zero
+
+    migrate!
+
+    expect(stages.count).to eq 6
+    expect(stages.all.pluck(:name))
+      .to match_array %w[test build deploy test:1 test:2 deploy]
+    expect(stages.where(pipeline_id: 1).order(:id).pluck(:name))
+      .to eq %w[test build deploy]
+    expect(stages.where(pipeline_id: 2).order(:id).pluck(:name))
+      .to eq %w[test:1 test:2 deploy]
+    expect(stages.where(pipeline_id: 3).count).to be_zero
+    expect(stages.where(project_id: 789).count).to be_zero
+  end
+end
diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb
index 7254e6b357a..2bd5af25847 100644
--- a/spec/services/ci/retry_build_service_spec.rb
+++ b/spec/services/ci/retry_build_service_spec.rb
@@ -18,11 +18,12 @@ describe Ci::RetryBuildService, :services do
        updated_at started_at finished_at queued_at erased_by
        erased_at auto_canceled_by].freeze
 
+  # TODO, move stage_id accessor to CLONE_ACCESSOR in a follow-up MR.
   IGNORE_ACCESSORS =
     %i[type lock_version target_url base_tags
        commit_id deployments erased_by_id last_deployment project_id
        runner_id tag_taggings taggings tags trigger_request_id
-       user_id auto_canceled_by_id retried].freeze
+       user_id auto_canceled_by_id retried stage_id].freeze
 
   shared_examples 'build duplication' do
     let(:build) do
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index f800c5bcb07..8b8fbf6e862 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -56,6 +56,7 @@ RSpec.configure do |config|
   config.include StubGitlabCalls
   config.include StubGitlabData
   config.include ApiHelpers, :api
+  config.include MigrationsHelpers, :migration
 
   config.infer_spec_type_from_file_location!
 
@@ -97,6 +98,17 @@ RSpec.configure do |config|
     Sidekiq.redis(&:flushall)
   end
 
+  config.around(:example, :migration) do |example|
+    begin
+      ActiveRecord::Migrator
+        .migrate(migrations_paths, previous_migration.version)
+
+      example.run
+    ensure
+      ActiveRecord::Migrator.migrate(migrations_paths)
+    end
+  end
+
   config.around(:each, :nested_groups) do |example|
     example.run if Group.supports_nested_groups?
   end
diff --git a/spec/support/db_cleaner.rb b/spec/support/db_cleaner.rb
index 6f31828b825..7f5769209bb 100644
--- a/spec/support/db_cleaner.rb
+++ b/spec/support/db_cleaner.rb
@@ -19,6 +19,10 @@ RSpec.configure do |config|
     DatabaseCleaner.strategy = :truncation
   end
 
+  config.before(:each, :migration) do
+    DatabaseCleaner.strategy = :truncation
+  end
+
   config.before(:each) do
     DatabaseCleaner.start
   end
diff --git a/spec/support/migrations_helpers.rb b/spec/support/migrations_helpers.rb
new file mode 100644
index 00000000000..91fbb4eaf48
--- /dev/null
+++ b/spec/support/migrations_helpers.rb
@@ -0,0 +1,29 @@
+module MigrationsHelpers
+  def table(name)
+    Class.new(ActiveRecord::Base) { self.table_name = name }
+  end
+
+  def migrations_paths
+    ActiveRecord::Migrator.migrations_paths
+  end
+
+  def table_exists?(name)
+    ActiveRecord::Base.connection.table_exists?(name)
+  end
+
+  def migrations
+    ActiveRecord::Migrator.migrations(migrations_paths)
+  end
+
+  def previous_migration
+    migrations.each_cons(2) do |previous, migration|
+      break previous if migration.name == described_class.name
+    end
+  end
+
+  def migrate!
+    ActiveRecord::Migrator.up(migrations_paths) do |migration|
+      migration.name == described_class.name
+    end
+  end
+end