From b21ee2ee36e1aaddbe0b3541a8cac5f117143b66 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 11:29:04 +0200 Subject: Add initial stage_id background migration files --- ...0170628080858_migrate_stage_id_reference_in_background.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb (limited to 'db') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb new file mode 100644 index 00000000000..2eaa798d0aa --- /dev/null +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -0,0 +1,12 @@ +class MigrateStageIdReferenceInBackground < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + end + + def down + # noop + end +end -- cgit v1.2.1 From 98992c4e4b3231a99eb5ff17c44e96fe79a6cff2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 12:01:52 +0200 Subject: Add initial build stage_id ref background migration --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'db') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 2eaa798d0aa..44bac4a8cc7 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -3,7 +3,17 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration DOWNTIME = false + disable_ddl_transaction! + + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + end + def up + Build.find_each do |build| + BackgroundMigrationWorker + .perform_async('MigrateBuildStageIdReference', [build.id]) + end end def down -- cgit v1.2.1 From 6209ff671fdd025be31f9dcaf208a71b6ec2907d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 12:20:45 +0200 Subject: Update `db/schema.rb` with a new schema version --- db/schema.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'db') diff --git a/db/schema.rb b/db/schema.rb index 8c7440ee610..3a45e0fe562 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170622162730) do +ActiveRecord::Schema.define(version: 20170628080858) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" -- cgit v1.2.1 From 02bb40e2acd7b1838e47e1a2f8b9288e42e6ca53 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 12:21:25 +0200 Subject: Find builds that require a migration in batches --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 44bac4a8cc7..6b326bc0b69 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -2,6 +2,8 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers DOWNTIME = false + BATCH_SIZE = 10000 + MIGRATION = 'MigrateBuildStageIdReference'.freeze disable_ddl_transaction! @@ -10,9 +12,10 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration end def up - Build.find_each do |build| - BackgroundMigrationWorker - .perform_async('MigrateBuildStageIdReference', [build.id]) + Build.find_in_batches(batch_size: BATCH_SIZE).with_index do |builds, batch| + migrations = builds.map { |build| [MIGRATION, [build.id]] } + + BackgroundMigrationWorker.perform_bulk(*migrations) end end -- cgit v1.2.1 From 5292eb651e1e3595e409a4c216eb0be3445a9319 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 12:23:00 +0200 Subject: Schedule background migration only when it is needed --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 6b326bc0b69..bfeb09f6da1 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -12,11 +12,13 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration end def up - Build.find_in_batches(batch_size: BATCH_SIZE).with_index do |builds, batch| - migrations = builds.map { |build| [MIGRATION, [build.id]] } + Build.where(stage_id: nil) + .find_in_batches(batch_size: BATCH_SIZE) + .with_index do |builds, batch| + migrations = builds.map { |build| [MIGRATION, [build.id]] } - BackgroundMigrationWorker.perform_bulk(*migrations) - end + BackgroundMigrationWorker.perform_bulk(*migrations) + end end def down -- cgit v1.2.1 From 187dd1005cd92c530146d7f5b0a89b368b09c3e9 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 15:24:53 +0200 Subject: Add specs for delayed stage_id background migrations --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index bfeb09f6da1..a73456af386 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -15,9 +15,10 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration Build.where(stage_id: nil) .find_in_batches(batch_size: BATCH_SIZE) .with_index do |builds, batch| - migrations = builds.map { |build| [MIGRATION, [build.id]] } - - BackgroundMigrationWorker.perform_bulk(*migrations) + builds.each do |build| + schedule = (batch - 1) * 5.minutes + BackgroundMigrationWorker.perform_at(schedule, MIGRATION, [build.id]) + end end end -- cgit v1.2.1 From 69736f3927160bd362e165b4cd9e78912a3c30c0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 12:10:29 +0200 Subject: Use ActiveRecord 5 batching to schedule bg migration --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index a73456af386..9e95216b35a 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -13,10 +13,9 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration def up Build.where(stage_id: nil) - .find_in_batches(batch_size: BATCH_SIZE) - .with_index do |builds, batch| - builds.each do |build| - schedule = (batch - 1) * 5.minutes + .in_batches(of: BATCH_SIZE) do |relation, index| + schedule = index * 5.minutes + relation.each do |build| BackgroundMigrationWorker.perform_at(schedule, MIGRATION, [build.id]) end end -- cgit v1.2.1 From 8bd9cb6c87a1a1c51830360fe1aa1a228f9c768e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 12:14:54 +0200 Subject: Perform stage_id ref backgound migration in bulks --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 9e95216b35a..c54e8bde095 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -15,9 +15,9 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration Build.where(stage_id: nil) .in_batches(of: BATCH_SIZE) do |relation, index| schedule = index * 5.minutes - relation.each do |build| - BackgroundMigrationWorker.perform_at(schedule, MIGRATION, [build.id]) - end + jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } + + BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) end end -- cgit v1.2.1 From 42556419c99522e921299a3e247f115f580be7f1 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 12:26:37 +0200 Subject: Improve specs for background stage_id ref migration --- ...20170628080858_migrate_stage_id_reference_in_background.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index c54e8bde095..1d95fc62c87 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -12,13 +12,12 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration end def up - Build.where(stage_id: nil) - .in_batches(of: BATCH_SIZE) do |relation, index| - schedule = index * 5.minutes - jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } + Build.where(stage_id: nil).in_batches(of: BATCH_SIZE) do |relation, index| + schedule = index * 5.minutes + jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } - BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) - end + BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) + end end def down -- cgit v1.2.1 From 134f204ed8f5dd80e44463338ae93f3d905ca7af Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 30 Jun 2017 13:03:47 +0200 Subject: Do not override original AR5 batching interface --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 1d95fc62c87..30849ea1361 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -12,9 +12,12 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration end def up - Build.where(stage_id: nil).in_batches(of: BATCH_SIZE) do |relation, index| - schedule = index * 5.minutes + index = 1 + + Build.where(stage_id: nil).in_batches(of: BATCH_SIZE) do |relation| jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } + schedule = index * 5.minutes + index += 1 BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) end -- cgit v1.2.1 From 9c7c95c768cd5294dd085c2fc2425fae91c4c689 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 30 Jun 2017 14:22:23 +0200 Subject: Add initial changes for stages statuses migration --- .../20170630105320_add_status_to_ci_stages.rb | 9 +++ .../20170630111158_migrate_stages_statuses.rb | 76 ++++++++++++++++++++++ db/schema.rb | 3 +- 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20170630105320_add_status_to_ci_stages.rb create mode 100644 db/post_migrate/20170630111158_migrate_stages_statuses.rb (limited to 'db') diff --git a/db/migrate/20170630105320_add_status_to_ci_stages.rb b/db/migrate/20170630105320_add_status_to_ci_stages.rb new file mode 100644 index 00000000000..d497a61a959 --- /dev/null +++ b/db/migrate/20170630105320_add_status_to_ci_stages.rb @@ -0,0 +1,9 @@ +class AddStatusToCiStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :ci_stages, :status, :integer + end +end diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb new file mode 100644 index 00000000000..b4b76893595 --- /dev/null +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -0,0 +1,76 @@ +class MigrateStagesStatuses < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + + scope :relevant, -> do + where(status: %w[pending running success failed canceled skipped manual]) + end + + scope :created, -> { where(status: 'created') } + scope :running, -> { where(status: 'running') } + scope :pending, -> { where(status: 'pending') } + scope :success, -> { where(status: 'success') } + scope :failed, -> { where(status: 'failed') } + scope :canceled, -> { where(status: 'canceled') } + scope :skipped, -> { where(status: 'skipped') } + scope :manual, -> { where(status: 'manual') } + + scope :failed_but_allowed, -> do + where(allow_failure: true, status: [:failed, :canceled]) + end + + scope :exclude_ignored, -> do + where("allow_failure = ? OR status IN (?)", + false, all_state_names - [:failed, :canceled, :manual]) + end + + def status_sql + scope_relevant = relevant.exclude_ignored + scope_warnings = relevant.failed_but_allowed + + builds = scope_relevant.select('count(*)').to_sql + created = scope_relevant.created.select('count(*)').to_sql + success = scope_relevant.success.select('count(*)').to_sql + manual = scope_relevant.manual.select('count(*)').to_sql + pending = scope_relevant.pending.select('count(*)').to_sql + running = scope_relevant.running.select('count(*)').to_sql + skipped = scope_relevant.skipped.select('count(*)').to_sql + canceled = scope_relevant.canceled.select('count(*)').to_sql + warnings = scope_warnings.select('count(*) > 0').to_sql + + "(CASE + WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN 'success' + WHEN (#{builds})=(#{skipped}) THEN 'skipped' + WHEN (#{builds})=(#{success}) THEN 'success' + WHEN (#{builds})=(#{created}) THEN 'created' + WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success' + WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled' + WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' + WHEN (#{running})+(#{pending})>0 THEN 'running' + WHEN (#{manual})>0 THEN 'manual' + WHEN (#{created})>0 THEN 'running' + ELSE 'failed' + END)" + end + end + + def up + execute <<-SQL.strip_heredoc + SQL + end + + def down + execute <<-SQL.strip_heredoc + UPDATE ci_stages SET status = null + SQL + end + + private + +end diff --git a/db/schema.rb b/db/schema.rb index 8c7440ee610..f34dd32fb74 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170622162730) do +ActiveRecord::Schema.define(version: 20170630111158) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -337,6 +337,7 @@ ActiveRecord::Schema.define(version: 20170622162730) do t.datetime "created_at" t.datetime "updated_at" t.string "name" + t.integer "status" end add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", using: :btree -- cgit v1.2.1 From f6966d96ec5941db364a2c8d9d2d80d3aa7d20f2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Jul 2017 13:02:51 +0200 Subject: Reduce a delay between stage_id scheduled migrations --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'db') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 30849ea1361..ebec4cb6bb7 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -16,7 +16,7 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration Build.where(stage_id: nil).in_batches(of: BATCH_SIZE) do |relation| jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } - schedule = index * 5.minutes + schedule = index * 2.minutes index += 1 BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) -- cgit v1.2.1 From 7103c4a707157594c261ba2f68fbb649ca4df769 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Jul 2017 09:20:18 +0200 Subject: Extend stages statuses migration --- .../20170630111158_migrate_stages_statuses.rb | 55 +++++++++++++--------- 1 file changed, 34 insertions(+), 21 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index b4b76893595..8c6de84adf5 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -5,13 +5,17 @@ class MigrateStagesStatuses < ActiveRecord::Migration disable_ddl_transaction! + STATUSES = { created: 0, pending: 1, running: 2, success: 3, + failed: 4, canceled: 5, skipped: 6, manual: 7 } + + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' + end + class Build < ActiveRecord::Base self.table_name = 'ci_builds' - scope :relevant, -> do - where(status: %w[pending running success failed canceled skipped manual]) - end - + scope :latest, -> { where(retried: [false, nil]) } scope :created, -> { where(status: 'created') } scope :running, -> { where(status: 'running') } scope :pending, -> { where(status: 'pending') } @@ -27,12 +31,12 @@ class MigrateStagesStatuses < ActiveRecord::Migration scope :exclude_ignored, -> do where("allow_failure = ? OR status IN (?)", - false, all_state_names - [:failed, :canceled, :manual]) + false, %w[created pending running success skipped]) end - def status_sql - scope_relevant = relevant.exclude_ignored - scope_warnings = relevant.failed_but_allowed + def self.status_sql + scope_relevant = latest.exclude_ignored + scope_warnings = latest.failed_but_allowed builds = scope_relevant.select('count(*)').to_sql created = scope_relevant.created.select('count(*)').to_sql @@ -45,24 +49,33 @@ class MigrateStagesStatuses < ActiveRecord::Migration warnings = scope_warnings.select('count(*) > 0').to_sql "(CASE - WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN 'success' - WHEN (#{builds})=(#{skipped}) THEN 'skipped' - WHEN (#{builds})=(#{success}) THEN 'success' - WHEN (#{builds})=(#{created}) THEN 'created' - WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success' - WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled' - WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' - WHEN (#{running})+(#{pending})>0 THEN 'running' - WHEN (#{manual})>0 THEN 'manual' - WHEN (#{created})>0 THEN 'running' - ELSE 'failed' + WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} + WHEN (#{builds})=(#{skipped}) THEN #{STATUSES[:skipped]} + WHEN (#{builds})=(#{success}) THEN #{STATUSES[:success]} + WHEN (#{builds})=(#{created}) THEN #{STATUSES[:created]} + WHEN (#{builds})=(#{success})+(#{skipped}) THEN #{STATUSES[:success]} + WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN #{STATUSES[:canceled]} + WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN #{STATUSES[:pending]} + WHEN (#{running})+(#{pending})>0 THEN '#{STATUSES[:running]} + WHEN (#{manual})>0 THEN #{STATUSES[:manual]} + WHEN (#{created})>0 THEN #{STATUSES[:running]} + ELSE #{STATUSES[:failed]} END)" end end def up - execute <<-SQL.strip_heredoc - SQL + Stage.all.in_batches(of: 10000) do |relation| + status_sql = Build + .where('ci_builds.commit_id = ci_stages.pipeline_id') + .where('ci_builds.stage = ci_stages.name') + .status_sql + + execute <<-SQL.strip_heredoc + UPDATE ci_stages SET status = #{status_sql} + WHERE id = (#{relation.select(:id).to_sql}) + SQL + end end def down -- cgit v1.2.1 From d60ce6e9f44eba769a6ad595014ae96095169dd2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Jul 2017 13:43:26 +0200 Subject: Implement initial working stages statuses migration --- .../20170630111158_migrate_stages_statuses.rb | 46 ++++++++++------------ 1 file changed, 20 insertions(+), 26 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index 8c6de84adf5..62542ed0001 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -48,34 +48,31 @@ class MigrateStagesStatuses < ActiveRecord::Migration canceled = scope_relevant.canceled.select('count(*)').to_sql warnings = scope_warnings.select('count(*) > 0').to_sql - "(CASE - WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} - WHEN (#{builds})=(#{skipped}) THEN #{STATUSES[:skipped]} - WHEN (#{builds})=(#{success}) THEN #{STATUSES[:success]} - WHEN (#{builds})=(#{created}) THEN #{STATUSES[:created]} - WHEN (#{builds})=(#{success})+(#{skipped}) THEN #{STATUSES[:success]} - WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN #{STATUSES[:canceled]} - WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN #{STATUSES[:pending]} - WHEN (#{running})+(#{pending})>0 THEN '#{STATUSES[:running]} - WHEN (#{manual})>0 THEN #{STATUSES[:manual]} - WHEN (#{created})>0 THEN #{STATUSES[:running]} - ELSE #{STATUSES[:failed]} - END)" + <<-SQL.strip_heredoc + (CASE + WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]} + WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]} + WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]} + WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]} + WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]} + WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]} + WHEN (#{created}) > 0 THEN #{STATUSES[:running]} + ELSE #{STATUSES[:failed]} + END) + SQL end end def up - Stage.all.in_batches(of: 10000) do |relation| - status_sql = Build - .where('ci_builds.commit_id = ci_stages.pipeline_id') - .where('ci_builds.stage = ci_stages.name') - .status_sql + status_sql = Build + .where('ci_builds.commit_id = ci_stages.pipeline_id') + .where('ci_builds.stage = ci_stages.name') + .status_sql - execute <<-SQL.strip_heredoc - UPDATE ci_stages SET status = #{status_sql} - WHERE id = (#{relation.select(:id).to_sql}) - SQL - end + update_column_in_batches(:ci_stages, :status, Arel.sql("(#{status_sql})")) end def down @@ -83,7 +80,4 @@ class MigrateStagesStatuses < ActiveRecord::Migration UPDATE ci_stages SET status = null SQL end - - private - end -- cgit v1.2.1 From 93d217bda639b94c129afd71343e429f935a4ada Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Jul 2017 16:58:35 +0200 Subject: Migrate only old stages without status that is set --- db/post_migrate/20170630111158_migrate_stages_statuses.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'db') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index 62542ed0001..9dac91960ff 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -72,7 +72,9 @@ class MigrateStagesStatuses < ActiveRecord::Migration .where('ci_builds.stage = ci_stages.name') .status_sql - update_column_in_batches(:ci_stages, :status, Arel.sql("(#{status_sql})")) + update_column_in_batches(:ci_stages, :status, Arel.sql("(#{status_sql})")) do |table, query| + query.where(table[:status].eq(nil)) + end end def down -- cgit v1.2.1 From c5f1e1a70bd79b36fe8cfda75b7366dd8ee90d66 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Jul 2017 09:11:15 +0200 Subject: Disable statement timeout in stages statuses migration --- db/post_migrate/20170630111158_migrate_stages_statuses.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index 9dac91960ff..c0a5294720d 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -8,10 +8,6 @@ class MigrateStagesStatuses < ActiveRecord::Migration STATUSES = { created: 0, pending: 1, running: 2, success: 3, failed: 4, canceled: 5, skipped: 6, manual: 7 } - class Stage < ActiveRecord::Base - self.table_name = 'ci_stages' - end - class Build < ActiveRecord::Base self.table_name = 'ci_builds' @@ -67,6 +63,8 @@ class MigrateStagesStatuses < ActiveRecord::Migration end def up + disable_statement_timeout + status_sql = Build .where('ci_builds.commit_id = ci_stages.pipeline_id') .where('ci_builds.stage = ci_stages.name') @@ -78,6 +76,8 @@ class MigrateStagesStatuses < ActiveRecord::Migration end def down + disable_statement_timeout + execute <<-SQL.strip_heredoc UPDATE ci_stages SET status = null SQL -- cgit v1.2.1 From 6c477d5b9496829eb5cb56ef32a0dd813be7dc16 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Jul 2017 10:54:48 +0200 Subject: Move stages status migration to the background worker --- .../20170630111158_migrate_stages_statuses.rb | 71 +++------------------- 1 file changed, 10 insertions(+), 61 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index c0a5294720d..2bc067e5d90 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -5,73 +5,22 @@ class MigrateStagesStatuses < ActiveRecord::Migration disable_ddl_transaction! - STATUSES = { created: 0, pending: 1, running: 2, success: 3, - failed: 4, canceled: 5, skipped: 6, manual: 7 } + BATCH_SIZE = 10000 + MIGRATION = 'MigrateStageStatus'.freeze - class Build < ActiveRecord::Base - self.table_name = 'ci_builds' - - scope :latest, -> { where(retried: [false, nil]) } - scope :created, -> { where(status: 'created') } - scope :running, -> { where(status: 'running') } - scope :pending, -> { where(status: 'pending') } - scope :success, -> { where(status: 'success') } - scope :failed, -> { where(status: 'failed') } - scope :canceled, -> { where(status: 'canceled') } - scope :skipped, -> { where(status: 'skipped') } - scope :manual, -> { where(status: 'manual') } - - scope :failed_but_allowed, -> do - where(allow_failure: true, status: [:failed, :canceled]) - end - - scope :exclude_ignored, -> do - where("allow_failure = ? OR status IN (?)", - false, %w[created pending running success skipped]) - end - - def self.status_sql - scope_relevant = latest.exclude_ignored - scope_warnings = latest.failed_but_allowed - - builds = scope_relevant.select('count(*)').to_sql - created = scope_relevant.created.select('count(*)').to_sql - success = scope_relevant.success.select('count(*)').to_sql - manual = scope_relevant.manual.select('count(*)').to_sql - pending = scope_relevant.pending.select('count(*)').to_sql - running = scope_relevant.running.select('count(*)').to_sql - skipped = scope_relevant.skipped.select('count(*)').to_sql - canceled = scope_relevant.canceled.select('count(*)').to_sql - warnings = scope_warnings.select('count(*) > 0').to_sql - - <<-SQL.strip_heredoc - (CASE - WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]} - WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]} - WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]} - WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]} - WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]} - WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]} - WHEN (#{created}) > 0 THEN #{STATUSES[:running]} - ELSE #{STATUSES[:failed]} - END) - SQL - end + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' end def up - disable_statement_timeout + index = 1 - status_sql = Build - .where('ci_builds.commit_id = ci_stages.pipeline_id') - .where('ci_builds.stage = ci_stages.name') - .status_sql + Stage.where(status: nil).in_batches(of: BATCH_SIZE) do |relation| + jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } + schedule = index * 5.minutes + index += 1 - update_column_in_batches(:ci_stages, :status, Arel.sql("(#{status_sql})")) do |table, query| - query.where(table[:status].eq(nil)) + BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) end end -- cgit v1.2.1 From 7082530d555ad98fede2823d2123622abaf1c3a3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 10 Jul 2017 15:42:19 +0200 Subject: Schedule stages statuses bg migrations in batches --- db/post_migrate/20170630111158_migrate_stages_statuses.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index 2bc067e5d90..1641e550480 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -6,21 +6,22 @@ class MigrateStagesStatuses < ActiveRecord::Migration disable_ddl_transaction! BATCH_SIZE = 10000 + RANGE_SIZE = 1000 MIGRATION = 'MigrateStageStatus'.freeze class Stage < ActiveRecord::Base self.table_name = 'ci_stages' + include ::EachBatch end def up - index = 1 + Stage.where(status: nil).each_batch(of: BATCH_SIZE) do |relation, index| + relation.each_batch(of: RANGE_SIZE) do |batch| + range = relation.pluck('MIN(id)', 'MAX(id)').first + schedule = index * 5.minutes - Stage.where(status: nil).in_batches(of: BATCH_SIZE) do |relation| - jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } - schedule = index * 5.minutes - index += 1 - - BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) + BackgroundMigrationWorker.perform_in(schedule, MIGRATION, range) + end end end -- cgit v1.2.1 From bb67b4749b5b4c62d4235c90dc0320967f850cdd Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 11 Jul 2017 14:31:04 +0200 Subject: Update version number of stages statuses migration --- .../20170630105320_add_status_to_ci_stages.rb | 9 ------ .../20170711145320_add_status_to_ci_stages.rb | 9 ++++++ .../20170630111158_migrate_stages_statuses.rb | 35 ---------------------- .../20170711145558_migrate_stages_statuses.rb | 35 ++++++++++++++++++++++ db/schema.rb | 2 +- 5 files changed, 45 insertions(+), 45 deletions(-) delete mode 100644 db/migrate/20170630105320_add_status_to_ci_stages.rb create mode 100644 db/migrate/20170711145320_add_status_to_ci_stages.rb delete mode 100644 db/post_migrate/20170630111158_migrate_stages_statuses.rb create mode 100644 db/post_migrate/20170711145558_migrate_stages_statuses.rb (limited to 'db') diff --git a/db/migrate/20170630105320_add_status_to_ci_stages.rb b/db/migrate/20170630105320_add_status_to_ci_stages.rb deleted file mode 100644 index d497a61a959..00000000000 --- a/db/migrate/20170630105320_add_status_to_ci_stages.rb +++ /dev/null @@ -1,9 +0,0 @@ -class AddStatusToCiStages < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - def change - add_column :ci_stages, :status, :integer - end -end diff --git a/db/migrate/20170711145320_add_status_to_ci_stages.rb b/db/migrate/20170711145320_add_status_to_ci_stages.rb new file mode 100644 index 00000000000..d497a61a959 --- /dev/null +++ b/db/migrate/20170711145320_add_status_to_ci_stages.rb @@ -0,0 +1,9 @@ +class AddStatusToCiStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :ci_stages, :status, :integer + end +end diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb deleted file mode 100644 index 1641e550480..00000000000 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ /dev/null @@ -1,35 +0,0 @@ -class MigrateStagesStatuses < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - disable_ddl_transaction! - - BATCH_SIZE = 10000 - RANGE_SIZE = 1000 - MIGRATION = 'MigrateStageStatus'.freeze - - class Stage < ActiveRecord::Base - self.table_name = 'ci_stages' - include ::EachBatch - end - - def up - Stage.where(status: nil).each_batch(of: BATCH_SIZE) do |relation, index| - relation.each_batch(of: RANGE_SIZE) do |batch| - range = relation.pluck('MIN(id)', 'MAX(id)').first - schedule = index * 5.minutes - - BackgroundMigrationWorker.perform_in(schedule, MIGRATION, range) - end - end - end - - def down - disable_statement_timeout - - execute <<-SQL.strip_heredoc - UPDATE ci_stages SET status = null - SQL - end -end diff --git a/db/post_migrate/20170711145558_migrate_stages_statuses.rb b/db/post_migrate/20170711145558_migrate_stages_statuses.rb new file mode 100644 index 00000000000..1641e550480 --- /dev/null +++ b/db/post_migrate/20170711145558_migrate_stages_statuses.rb @@ -0,0 +1,35 @@ +class MigrateStagesStatuses < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + BATCH_SIZE = 10000 + RANGE_SIZE = 1000 + MIGRATION = 'MigrateStageStatus'.freeze + + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' + include ::EachBatch + end + + def up + Stage.where(status: nil).each_batch(of: BATCH_SIZE) do |relation, index| + relation.each_batch(of: RANGE_SIZE) do |batch| + range = relation.pluck('MIN(id)', 'MAX(id)').first + schedule = index * 5.minutes + + BackgroundMigrationWorker.perform_in(schedule, MIGRATION, range) + end + end + end + + def down + disable_statement_timeout + + execute <<-SQL.strip_heredoc + UPDATE ci_stages SET status = null + SQL + end +end diff --git a/db/schema.rb b/db/schema.rb index e0c9e5efb33..3ef311f48d8 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170707184244) do +ActiveRecord::Schema.define(version: 20170711145558) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" -- cgit v1.2.1 From e389507650769304bd61f7a82431cc6c07feb364 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 20 Jul 2017 13:17:48 +0200 Subject: Add optimistic locking column to ci_stages table --- db/migrate/20170720111708_add_lock_version_to_ci_stages.rb | 9 +++++++++ db/schema.rb | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20170720111708_add_lock_version_to_ci_stages.rb (limited to 'db') diff --git a/db/migrate/20170720111708_add_lock_version_to_ci_stages.rb b/db/migrate/20170720111708_add_lock_version_to_ci_stages.rb new file mode 100644 index 00000000000..e1c4f033286 --- /dev/null +++ b/db/migrate/20170720111708_add_lock_version_to_ci_stages.rb @@ -0,0 +1,9 @@ +class AddLockVersionToCiStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :ci_stages, :lock_version, :integer + end +end diff --git a/db/schema.rb b/db/schema.rb index 862b2e21f4d..567ba4d061b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170717150329) do +ActiveRecord::Schema.define(version: 20170720111708) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -366,6 +366,7 @@ ActiveRecord::Schema.define(version: 20170717150329) do t.datetime "updated_at" t.string "name" t.integer "status" + t.integer "lock_version" end add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", using: :btree -- cgit v1.2.1 From 13a15e7009e292919109ea911640a627dbd0e327 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 31 Jul 2017 12:23:04 +0200 Subject: Use update_column_in_batches helper in stages migration --- db/post_migrate/20170711145558_migrate_stages_statuses.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20170711145558_migrate_stages_statuses.rb b/db/post_migrate/20170711145558_migrate_stages_statuses.rb index 1641e550480..5a24fb1307f 100644 --- a/db/post_migrate/20170711145558_migrate_stages_statuses.rb +++ b/db/post_migrate/20170711145558_migrate_stages_statuses.rb @@ -28,8 +28,6 @@ class MigrateStagesStatuses < ActiveRecord::Migration def down disable_statement_timeout - execute <<-SQL.strip_heredoc - UPDATE ci_stages SET status = null - SQL + update_column_in_batches(:ci_stages, :status, nil) end end -- cgit v1.2.1 From 53403399577bdca0e8f0886fa62ce0e75c14a8e0 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 2 Aug 2017 04:11:35 +0200 Subject: Add UUID Storage to Project --- ...20170802013652_add_storage_fields_to_project.rb | 39 ++++++++++++++++++++++ db/schema.rb | 3 ++ 2 files changed, 42 insertions(+) create mode 100644 db/migrate/20170802013652_add_storage_fields_to_project.rb (limited to 'db') diff --git a/db/migrate/20170802013652_add_storage_fields_to_project.rb b/db/migrate/20170802013652_add_storage_fields_to_project.rb new file mode 100644 index 00000000000..269103cd472 --- /dev/null +++ b/db/migrate/20170802013652_add_storage_fields_to_project.rb @@ -0,0 +1,39 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddStorageFieldsToProject < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index", "remove_concurrent_index" or + # "add_column_with_default" you must disable the use of transactions + # as these methods can not run in an existing transaction. + # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure + # that either of them is the _only_ method called in the migration, + # any other changes should go in a separate migration. + # This ensures that upon failure _only_ the index creation or removing fails + # and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + disable_ddl_transaction! + + def up + # rubocop:disable Migration/AddColumnWithDefaultToLargeTable + add_column :projects, :uuid, :uuid + add_column_with_default :projects, :storage_version, :integer, default: 0, limit: 1 + add_concurrent_index :projects, :uuid + end + + def down + remove_column :projects, :uuid + remove_column :projects, :storage_version + end +end diff --git a/db/schema.rb b/db/schema.rb index c31bff3a8f2..dcd9532e4be 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1208,6 +1208,8 @@ ActiveRecord::Schema.define(version: 20170820100558) do t.datetime "last_repository_updated_at" t.string "ci_config_path" t.text "delete_error" + t.uuid "uuid" + t.integer "storage_version", limit: 2, default: 0, null: false end add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree @@ -1224,6 +1226,7 @@ ActiveRecord::Schema.define(version: 20170820100558) do add_index "projects", ["pending_delete"], name: "index_projects_on_pending_delete", using: :btree add_index "projects", ["runners_token"], name: "index_projects_on_runners_token", using: :btree add_index "projects", ["star_count"], name: "index_projects_on_star_count", using: :btree + add_index "projects", ["uuid"], name: "index_projects_on_uuid", using: :btree add_index "projects", ["visibility_level"], name: "index_projects_on_visibility_level", using: :btree create_table "protected_branch_merge_access_levels", force: :cascade do |t| -- cgit v1.2.1 From 9e6fa996eab978506af1084b79a9c3f91f6d575b Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Fri, 4 Aug 2017 07:30:42 +0200 Subject: New storage is now "Hashed" instead of "UUID" --- ...20170802013652_add_storage_fields_to_project.rb | 24 ++-------------------- db/schema.rb | 5 ++--- 2 files changed, 4 insertions(+), 25 deletions(-) (limited to 'db') diff --git a/db/migrate/20170802013652_add_storage_fields_to_project.rb b/db/migrate/20170802013652_add_storage_fields_to_project.rb index 269103cd472..a0815da0fcd 100644 --- a/db/migrate/20170802013652_add_storage_fields_to_project.rb +++ b/db/migrate/20170802013652_add_storage_fields_to_project.rb @@ -4,36 +4,16 @@ class AddStorageFieldsToProject < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers - # Set this constant to true if this migration requires downtime. DOWNTIME = false - - # When a migration requires downtime you **must** uncomment the following - # constant and define a short and easy to understand explanation as to why the - # migration requires downtime. - # DOWNTIME_REASON = '' - - # When using the methods "add_concurrent_index", "remove_concurrent_index" or - # "add_column_with_default" you must disable the use of transactions - # as these methods can not run in an existing transaction. - # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure - # that either of them is the _only_ method called in the migration, - # any other changes should go in a separate migration. - # This ensures that upon failure _only_ the index creation or removing fails - # and can be retried or reverted easily. - # - # To disable transactions uncomment the following line and remove these - # comments: disable_ddl_transaction! def up # rubocop:disable Migration/AddColumnWithDefaultToLargeTable - add_column :projects, :uuid, :uuid - add_column_with_default :projects, :storage_version, :integer, default: 0, limit: 1 - add_concurrent_index :projects, :uuid + add_column :projects, :storage_version, :integer, limit: 2 + add_concurrent_index :projects, :storage_version end def down - remove_column :projects, :uuid remove_column :projects, :storage_version end end diff --git a/db/schema.rb b/db/schema.rb index dcd9532e4be..5a85a00bb12 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1208,8 +1208,7 @@ ActiveRecord::Schema.define(version: 20170820100558) do t.datetime "last_repository_updated_at" t.string "ci_config_path" t.text "delete_error" - t.uuid "uuid" - t.integer "storage_version", limit: 2, default: 0, null: false + t.integer "storage_version", limit: 2 end add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree @@ -1226,7 +1225,7 @@ ActiveRecord::Schema.define(version: 20170820100558) do add_index "projects", ["pending_delete"], name: "index_projects_on_pending_delete", using: :btree add_index "projects", ["runners_token"], name: "index_projects_on_runners_token", using: :btree add_index "projects", ["star_count"], name: "index_projects_on_star_count", using: :btree - add_index "projects", ["uuid"], name: "index_projects_on_uuid", using: :btree + add_index "projects", ["storage_version"], name: "index_projects_on_storage_version", using: :btree add_index "projects", ["visibility_level"], name: "index_projects_on_visibility_level", using: :btree create_table "protected_branch_merge_access_levels", force: :cascade do |t| -- cgit v1.2.1 From 72250a4ed8978b32c2e12dd05fc6feb8132e4083 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Mon, 7 Aug 2017 10:53:30 +0200 Subject: Enable automatic hashed storage for new projects by application settings --- .../20170807071105_add_hashed_storage_to_settings.rb | 18 ++++++++++++++++++ db/schema.rb | 1 + 2 files changed, 19 insertions(+) create mode 100644 db/migrate/20170807071105_add_hashed_storage_to_settings.rb (limited to 'db') diff --git a/db/migrate/20170807071105_add_hashed_storage_to_settings.rb b/db/migrate/20170807071105_add_hashed_storage_to_settings.rb new file mode 100644 index 00000000000..0846557add8 --- /dev/null +++ b/db/migrate/20170807071105_add_hashed_storage_to_settings.rb @@ -0,0 +1,18 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddHashedStorageToSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column_with_default :application_settings, :hashed_storage_enabled, :boolean, default: false + end + + def down + remove_columns :application_settings, :hashed_storage_enabled + end +end diff --git a/db/schema.rb b/db/schema.rb index 5a85a00bb12..4d999ead607 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -128,6 +128,7 @@ ActiveRecord::Schema.define(version: 20170820100558) do t.integer "performance_bar_allowed_group_id" t.boolean "password_authentication_enabled" t.boolean "project_export_enabled", default: true, null: false + t.boolean "hashed_storage_enabled", default: false, null: false end create_table "audit_events", force: :cascade do |t| -- cgit v1.2.1 From e7a060321fed61085c7d70d23e9ea33825d1467f Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Mon, 7 Aug 2017 11:07:42 +0200 Subject: Moving away from the "extend" based factory to a more traditional one. Using `extend` dynamically can lead to bad performance as it invalidates the method's cache. --- db/migrate/20170802013652_add_storage_fields_to_project.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'db') diff --git a/db/migrate/20170802013652_add_storage_fields_to_project.rb b/db/migrate/20170802013652_add_storage_fields_to_project.rb index a0815da0fcd..e99ae53ef11 100644 --- a/db/migrate/20170802013652_add_storage_fields_to_project.rb +++ b/db/migrate/20170802013652_add_storage_fields_to_project.rb @@ -8,7 +8,6 @@ class AddStorageFieldsToProject < ActiveRecord::Migration disable_ddl_transaction! def up - # rubocop:disable Migration/AddColumnWithDefaultToLargeTable add_column :projects, :storage_version, :integer, limit: 2 add_concurrent_index :projects, :storage_version end -- cgit v1.2.1 From fff5ebdcae6794de35f8eaff15217d8643c83686 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 9 Aug 2017 08:41:32 +0200 Subject: Removed some useless code, codestyle changes and removed an index --- db/migrate/20170802013652_add_storage_fields_to_project.rb | 2 -- db/schema.rb | 1 - 2 files changed, 3 deletions(-) (limited to 'db') diff --git a/db/migrate/20170802013652_add_storage_fields_to_project.rb b/db/migrate/20170802013652_add_storage_fields_to_project.rb index e99ae53ef11..c2381a9d0b2 100644 --- a/db/migrate/20170802013652_add_storage_fields_to_project.rb +++ b/db/migrate/20170802013652_add_storage_fields_to_project.rb @@ -5,11 +5,9 @@ class AddStorageFieldsToProject < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers DOWNTIME = false - disable_ddl_transaction! def up add_column :projects, :storage_version, :integer, limit: 2 - add_concurrent_index :projects, :storage_version end def down diff --git a/db/schema.rb b/db/schema.rb index 4d999ead607..cd488630237 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1226,7 +1226,6 @@ ActiveRecord::Schema.define(version: 20170820100558) do add_index "projects", ["pending_delete"], name: "index_projects_on_pending_delete", using: :btree add_index "projects", ["runners_token"], name: "index_projects_on_runners_token", using: :btree add_index "projects", ["star_count"], name: "index_projects_on_star_count", using: :btree - add_index "projects", ["storage_version"], name: "index_projects_on_storage_version", using: :btree add_index "projects", ["visibility_level"], name: "index_projects_on_visibility_level", using: :btree create_table "protected_branch_merge_access_levels", force: :cascade do |t| -- cgit v1.2.1