From 90a99ff3aaa8f70b21369f976b075174e71f0a93 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 29 Jun 2018 13:57:52 +0000 Subject: Add pipeline stages position clean-up migration --- ...stage-gb-stages-position-migration-clean-up.yml | 5 ++ ...0604123514_cleanup_stages_position_migration.rb | 43 +++++++++++++ doc/user/project/settings/import_export.md | 3 +- lib/gitlab/import_export.rb | 2 +- lib/tasks/gitlab/import_export.rake | 21 ++++--- .../import_export/test_project_export.tar.gz | Bin 343091 -> 343136 bytes spec/fixtures/exported-project.gz | Bin 2560 -> 0 bytes spec/lib/gitlab/import_export/importer_spec.rb | 4 +- .../cleanup_stages_position_migration_spec.rb | 67 +++++++++++++++++++++ vendor/project_templates/express.tar.gz | Bin 4866 -> 4894 bytes vendor/project_templates/rails.tar.gz | Bin 25151 -> 25182 bytes vendor/project_templates/spring.tar.gz | Bin 49430 -> 49476 bytes 12 files changed, 134 insertions(+), 11 deletions(-) create mode 100644 changelogs/unreleased/backstage-gb-stages-position-migration-clean-up.yml create mode 100644 db/post_migrate/20180604123514_cleanup_stages_position_migration.rb delete mode 100644 spec/fixtures/exported-project.gz create mode 100644 spec/migrations/cleanup_stages_position_migration_spec.rb diff --git a/changelogs/unreleased/backstage-gb-stages-position-migration-clean-up.yml b/changelogs/unreleased/backstage-gb-stages-position-migration-clean-up.yml new file mode 100644 index 00000000000..d2ada88870b --- /dev/null +++ b/changelogs/unreleased/backstage-gb-stages-position-migration-clean-up.yml @@ -0,0 +1,5 @@ +--- +title: Fully migrate pipeline stages position +merge_request: 19369 +author: +type: performance diff --git a/db/post_migrate/20180604123514_cleanup_stages_position_migration.rb b/db/post_migrate/20180604123514_cleanup_stages_position_migration.rb new file mode 100644 index 00000000000..73c23dffca0 --- /dev/null +++ b/db/post_migrate/20180604123514_cleanup_stages_position_migration.rb @@ -0,0 +1,43 @@ +class CleanupStagesPositionMigration < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + TMP_INDEX_NAME = 'tmp_id_stage_position_partial_null_index'.freeze + + disable_ddl_transaction! + + class Stages < ActiveRecord::Base + include EachBatch + self.table_name = 'ci_stages' + end + + def up + disable_statement_timeout + + Gitlab::BackgroundMigration.steal('MigrateStageIndex') + + unless index_exists_by_name?(:ci_stages, TMP_INDEX_NAME) + add_concurrent_index(:ci_stages, :id, where: 'position IS NULL', name: TMP_INDEX_NAME) + end + + migratable = <<~SQL + position IS NULL AND EXISTS ( + SELECT 1 FROM ci_builds WHERE stage_id = ci_stages.id AND stage_idx IS NOT NULL + ) + SQL + + Stages.where(migratable).each_batch(of: 1000) do |batch| + batch.pluck(:id).each do |stage| + Gitlab::BackgroundMigration::MigrateStageIndex.new.perform(stage, stage) + end + end + + remove_concurrent_index_by_name(:ci_stages, TMP_INDEX_NAME) + end + + def down + if index_exists_by_name?(:ci_stages, TMP_INDEX_NAME) + remove_concurrent_index_by_name(:ci_stages, TMP_INDEX_NAME) + end + end +end diff --git a/doc/user/project/settings/import_export.md b/doc/user/project/settings/import_export.md index 9034a9b5179..f94671fcf87 100644 --- a/doc/user/project/settings/import_export.md +++ b/doc/user/project/settings/import_export.md @@ -32,7 +32,8 @@ with all their related data and be moved into a new GitLab instance. | GitLab version | Import/Export version | | ---------------- | --------------------- | -| 10.8 to current | 0.2.3 | +| 11.1 to current | 0.2.4 | +| 10.8 | 0.2.3 | | 10.4 | 0.2.2 | | 10.3 | 0.2.1 | | 10.0 | 0.2.0 | diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index b713fa7e1cd..53fe2f8e436 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.3'.freeze + VERSION = '0.2.4'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake index 44074397c05..900dbf7be24 100644 --- a/lib/tasks/gitlab/import_export.rake +++ b/lib/tasks/gitlab/import_export.rake @@ -10,15 +10,22 @@ namespace :gitlab do puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true) end - desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz' - task bump_test_version: :environment do - Dir.mktmpdir do |tmp_dir| - system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null") - File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') - system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null") + desc 'GitLab | Bumps the Import/Export version in fixtures and project templates' + task bump_version: :environment do + archives = Dir['vendor/project_templates/*.tar.gz'] + archives.push('spec/features/projects/import_export/test_project_export.tar.gz') + + archives.each do |archive| + raise ArgumentError unless File.exist?(archive) + + Dir.mktmpdir do |tmp_dir| + system("tar -zxf #{archive} -C #{tmp_dir} > /dev/null") + File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') + system("tar -zcvf #{archive} -C #{tmp_dir} . > /dev/null") + end end - puts "Updated to #{Gitlab::ImportExport.version}" + puts "Updated #{archives} to #{Gitlab::ImportExport.version}." end end end diff --git a/spec/features/projects/import_export/test_project_export.tar.gz b/spec/features/projects/import_export/test_project_export.tar.gz index 72ab2d71f35..ceba4dfec57 100644 Binary files a/spec/features/projects/import_export/test_project_export.tar.gz and b/spec/features/projects/import_export/test_project_export.tar.gz differ diff --git a/spec/fixtures/exported-project.gz b/spec/fixtures/exported-project.gz deleted file mode 100644 index bef7e2ff8ee..00000000000 Binary files a/spec/fixtures/exported-project.gz and /dev/null differ diff --git a/spec/lib/gitlab/import_export/importer_spec.rb b/spec/lib/gitlab/import_export/importer_spec.rb index 991e354f499..c074e61da26 100644 --- a/spec/lib/gitlab/import_export/importer_spec.rb +++ b/spec/lib/gitlab/import_export/importer_spec.rb @@ -4,14 +4,14 @@ describe Gitlab::ImportExport::Importer do let(:user) { create(:user) } let(:test_path) { "#{Dir.tmpdir}/importer_spec" } let(:shared) { project.import_export_shared } - let(:project) { create(:project, import_source: File.join(test_path, 'exported-project.gz')) } + let(:project) { create(:project, import_source: File.join(test_path, 'test_project_export.tar.gz')) } subject(:importer) { described_class.new(project) } before do allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path) FileUtils.mkdir_p(shared.export_path) - FileUtils.cp(Rails.root.join('spec', 'fixtures', 'exported-project.gz'), test_path) + FileUtils.cp(Rails.root.join('spec/features/projects/import_export/test_project_export.tar.gz'), test_path) allow(subject).to receive(:remove_import_file) end diff --git a/spec/migrations/cleanup_stages_position_migration_spec.rb b/spec/migrations/cleanup_stages_position_migration_spec.rb new file mode 100644 index 00000000000..dde5a777487 --- /dev/null +++ b/spec/migrations/cleanup_stages_position_migration_spec.rb @@ -0,0 +1,67 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20180604123514_cleanup_stages_position_migration.rb') + +describe CleanupStagesPositionMigration, :migration, :sidekiq, :redis do + let(:migration) { spy('migration') } + + before do + allow(Gitlab::BackgroundMigration::MigrateStageIndex) + .to receive(:new).and_return(migration) + end + + context 'when there are pending background migrations' do + it 'processes pending jobs synchronously' do + Sidekiq::Testing.disable! do + BackgroundMigrationWorker + .perform_in(2.minutes, 'MigrateStageIndex', [1, 1]) + BackgroundMigrationWorker + .perform_async('MigrateStageIndex', [1, 1]) + + migrate! + + expect(migration).to have_received(:perform).with(1, 1).twice + end + end + end + + context 'when there are no background migrations pending' do + it 'does nothing' do + Sidekiq::Testing.disable! do + migrate! + + expect(migration).not_to have_received(:perform) + end + end + end + + context 'when there are still unmigrated stages present' do + let(:stages) { table('ci_stages') } + let(:builds) { table('ci_builds') } + + let!(:entities) do + %w[build test broken].map do |name| + stages.create(name: name) + end + end + + before do + stages.update_all(position: nil) + + builds.create(name: 'unit', stage_id: entities.first.id, stage_idx: 1, ref: 'master') + builds.create(name: 'unit', stage_id: entities.second.id, stage_idx: 1, ref: 'master') + end + + it 'migrates stages sequentially for every stage' do + expect(stages.all).to all(have_attributes(position: nil)) + + migrate! + + expect(migration).to have_received(:perform) + .with(entities.first.id, entities.first.id) + expect(migration).to have_received(:perform) + .with(entities.second.id, entities.second.id) + expect(migration).not_to have_received(:perform) + .with(entities.third.id, entities.third.id) + end + end +end diff --git a/vendor/project_templates/express.tar.gz b/vendor/project_templates/express.tar.gz index 8dd5fa36987..fb357639a69 100644 Binary files a/vendor/project_templates/express.tar.gz and b/vendor/project_templates/express.tar.gz differ diff --git a/vendor/project_templates/rails.tar.gz b/vendor/project_templates/rails.tar.gz index 89337dc5c31..8454d2fc03b 100644 Binary files a/vendor/project_templates/rails.tar.gz and b/vendor/project_templates/rails.tar.gz differ diff --git a/vendor/project_templates/spring.tar.gz b/vendor/project_templates/spring.tar.gz index 31c90d0820f..55e25fdbe7c 100644 Binary files a/vendor/project_templates/spring.tar.gz and b/vendor/project_templates/spring.tar.gz differ -- cgit v1.2.1