Merge branch 'backstage/gb/stages-position-migration-clean-up' into 'master'

Add pipeline stages position clean-up migration

Closes #45956

See merge request gitlab-org/gitlab-ce!19369

12 files changed, 134 insertions, 11 deletions

diff --git a/changelogs/unreleased/backstage-gb-stages-position-migration-clean-up.yml b/changelogs/unreleased/backstage-gb-stages-position-migration-clean-up.yml
new file mode 100644
index 00000000000..d2ada88870b
--- /dev/null
+++ b/changelogs/unreleased/backstage-gb-stages-position-migration-clean-up.yml
@@ -0,0 +1,5 @@
+---
+title: Fully migrate pipeline stages position
+merge_request: 19369
+author:
+type: performance
diff --git a/db/post_migrate/20180604123514_cleanup_stages_position_migration.rb b/db/post_migrate/20180604123514_cleanup_stages_position_migration.rb
new file mode 100644
index 00000000000..73c23dffca0
--- /dev/null
+++ b/db/post_migrate/20180604123514_cleanup_stages_position_migration.rb
@@ -0,0 +1,43 @@
+class CleanupStagesPositionMigration < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  TMP_INDEX_NAME = 'tmp_id_stage_position_partial_null_index'.freeze
+
+  disable_ddl_transaction!
+
+  class Stages < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'ci_stages'
+  end
+
+  def up
+    disable_statement_timeout
+
+    Gitlab::BackgroundMigration.steal('MigrateStageIndex')
+
+    unless index_exists_by_name?(:ci_stages, TMP_INDEX_NAME)
+      add_concurrent_index(:ci_stages, :id, where: 'position IS NULL', name: TMP_INDEX_NAME)
+    end
+
+    migratable = <<~SQL
+      position IS NULL AND EXISTS (
+        SELECT 1 FROM ci_builds WHERE stage_id = ci_stages.id AND stage_idx IS NOT NULL
+      )
+    SQL
+
+    Stages.where(migratable).each_batch(of: 1000) do |batch|
+      batch.pluck(:id).each do |stage|
+        Gitlab::BackgroundMigration::MigrateStageIndex.new.perform(stage, stage)
+      end
+    end
+
+    remove_concurrent_index_by_name(:ci_stages, TMP_INDEX_NAME)
+  end
+
+  def down
+    if index_exists_by_name?(:ci_stages, TMP_INDEX_NAME)
+      remove_concurrent_index_by_name(:ci_stages, TMP_INDEX_NAME)
+    end
+  end
+end
diff --git a/doc/user/project/settings/import_export.md b/doc/user/project/settings/import_export.md
index 9034a9b5179..f94671fcf87 100644
--- a/doc/user/project/settings/import_export.md
+++ b/doc/user/project/settings/import_export.md
@@ -32,7 +32,8 @@ with all their related data and be moved into a new GitLab instance.
 
 | GitLab version   | Import/Export version |
 | ---------------- | --------------------- |
-| 10.8 to current  | 0.2.3                 |
+| 11.1 to current  | 0.2.4                 |
+| 10.8             | 0.2.3                 |
 | 10.4             | 0.2.2                 |
 | 10.3             | 0.2.1                 |
 | 10.0             | 0.2.0                 |
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index b713fa7e1cd..53fe2f8e436 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -3,7 +3,7 @@ module Gitlab
     extend self
 
     # For every version update, the version history in import_export.md has to be kept up to date.
-    VERSION = '0.2.3'.freeze
+    VERSION = '0.2.4'.freeze
     FILENAME_LIMIT = 50
 
     def export_path(relative_path:)
diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake
index 44074397c05..900dbf7be24 100644
--- a/lib/tasks/gitlab/import_export.rake
+++ b/lib/tasks/gitlab/import_export.rake
@@ -10,15 +10,22 @@ namespace :gitlab do
       puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true)
     end
 
-    desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz'
-    task bump_test_version: :environment do
-      Dir.mktmpdir do |tmp_dir|
-        system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null")
-        File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w')
-        system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null")
+    desc 'GitLab | Bumps the Import/Export version in fixtures and project templates'
+    task bump_version: :environment do
+      archives = Dir['vendor/project_templates/*.tar.gz']
+      archives.push('spec/features/projects/import_export/test_project_export.tar.gz')
+
+      archives.each do |archive|
+        raise ArgumentError unless File.exist?(archive)
+
+        Dir.mktmpdir do |tmp_dir|
+          system("tar -zxf #{archive} -C #{tmp_dir} > /dev/null")
+          File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w')
+          system("tar -zcvf #{archive} -C #{tmp_dir} . > /dev/null")
+        end
       end
 
-      puts "Updated to #{Gitlab::ImportExport.version}"
+      puts "Updated #{archives} to #{Gitlab::ImportExport.version}."
     end
   end
 end
diff --git a/spec/features/projects/import_export/test_project_export.tar.gz b/spec/features/projects/import_export/test_project_export.tar.gz
index 72ab2d71f35..ceba4dfec57 100644
--- a/spec/features/projects/import_export/test_project_export.tar.gz
+++ b/spec/features/projects/import_export/test_project_export.tar.gz
diff --git a/spec/fixtures/exported-project.gz b/spec/fixtures/exported-project.gz
deleted file mode 100644
index bef7e2ff8ee..00000000000
--- a/spec/fixtures/exported-project.gz
+++ /dev/null
diff --git a/spec/lib/gitlab/import_export/importer_spec.rb b/spec/lib/gitlab/import_export/importer_spec.rb
index 991e354f499..c074e61da26 100644
--- a/spec/lib/gitlab/import_export/importer_spec.rb
+++ b/spec/lib/gitlab/import_export/importer_spec.rb
@@ -4,14 +4,14 @@ describe Gitlab::ImportExport::Importer do
   let(:user) { create(:user) }
   let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
   let(:shared) { project.import_export_shared }
-  let(:project) { create(:project, import_source: File.join(test_path, 'exported-project.gz')) }
+  let(:project) { create(:project, import_source: File.join(test_path, 'test_project_export.tar.gz')) }
 
   subject(:importer) { described_class.new(project) }
 
   before do
     allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
     FileUtils.mkdir_p(shared.export_path)
-    FileUtils.cp(Rails.root.join('spec', 'fixtures', 'exported-project.gz'), test_path)
+    FileUtils.cp(Rails.root.join('spec/features/projects/import_export/test_project_export.tar.gz'), test_path)
     allow(subject).to receive(:remove_import_file)
   end
 
diff --git a/spec/migrations/cleanup_stages_position_migration_spec.rb b/spec/migrations/cleanup_stages_position_migration_spec.rb
new file mode 100644
index 00000000000..dde5a777487
--- /dev/null
+++ b/spec/migrations/cleanup_stages_position_migration_spec.rb
@@ -0,0 +1,67 @@
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20180604123514_cleanup_stages_position_migration.rb')
+
+describe CleanupStagesPositionMigration, :migration, :sidekiq, :redis do
+  let(:migration) { spy('migration') }
+
+  before do
+    allow(Gitlab::BackgroundMigration::MigrateStageIndex)
+      .to receive(:new).and_return(migration)
+  end
+
+  context 'when there are pending background migrations' do
+    it 'processes pending jobs synchronously' do
+      Sidekiq::Testing.disable! do
+        BackgroundMigrationWorker
+          .perform_in(2.minutes, 'MigrateStageIndex', [1, 1])
+        BackgroundMigrationWorker
+          .perform_async('MigrateStageIndex', [1, 1])
+
+        migrate!
+
+        expect(migration).to have_received(:perform).with(1, 1).twice
+      end
+    end
+  end
+
+  context 'when there are no background migrations pending' do
+    it 'does nothing' do
+      Sidekiq::Testing.disable! do
+        migrate!
+
+        expect(migration).not_to have_received(:perform)
+      end
+    end
+  end
+
+  context 'when there are still unmigrated stages present' do
+    let(:stages) { table('ci_stages') }
+    let(:builds) { table('ci_builds') }
+
+    let!(:entities) do
+      %w[build test broken].map do |name|
+        stages.create(name: name)
+      end
+    end
+
+    before do
+      stages.update_all(position: nil)
+
+      builds.create(name: 'unit', stage_id: entities.first.id, stage_idx: 1, ref: 'master')
+      builds.create(name: 'unit', stage_id: entities.second.id, stage_idx: 1, ref: 'master')
+    end
+
+    it 'migrates stages sequentially for every stage' do
+      expect(stages.all).to all(have_attributes(position: nil))
+
+      migrate!
+
+      expect(migration).to have_received(:perform)
+        .with(entities.first.id, entities.first.id)
+      expect(migration).to have_received(:perform)
+        .with(entities.second.id, entities.second.id)
+      expect(migration).not_to have_received(:perform)
+        .with(entities.third.id, entities.third.id)
+    end
+  end
+end
diff --git a/vendor/project_templates/express.tar.gz b/vendor/project_templates/express.tar.gz
index 8dd5fa36987..fb357639a69 100644
--- a/vendor/project_templates/express.tar.gz
+++ b/vendor/project_templates/express.tar.gz
diff --git a/vendor/project_templates/rails.tar.gz b/vendor/project_templates/rails.tar.gz
index 89337dc5c31..8454d2fc03b 100644
--- a/vendor/project_templates/rails.tar.gz
+++ b/vendor/project_templates/rails.tar.gz
diff --git a/vendor/project_templates/spring.tar.gz b/vendor/project_templates/spring.tar.gz
index 31c90d0820f..55e25fdbe7c 100644
--- a/vendor/project_templates/spring.tar.gz
+++ b/vendor/project_templates/spring.tar.gz