From 6c477d5b9496829eb5cb56ef32a0dd813be7dc16 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Jul 2017 10:54:48 +0200 Subject: Move stages status migration to the background worker --- .../20170630111158_migrate_stages_statuses.rb | 71 +++------------------- 1 file changed, 10 insertions(+), 61 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index c0a5294720d..2bc067e5d90 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -5,73 +5,22 @@ class MigrateStagesStatuses < ActiveRecord::Migration disable_ddl_transaction! - STATUSES = { created: 0, pending: 1, running: 2, success: 3, - failed: 4, canceled: 5, skipped: 6, manual: 7 } + BATCH_SIZE = 10000 + MIGRATION = 'MigrateStageStatus'.freeze - class Build < ActiveRecord::Base - self.table_name = 'ci_builds' - - scope :latest, -> { where(retried: [false, nil]) } - scope :created, -> { where(status: 'created') } - scope :running, -> { where(status: 'running') } - scope :pending, -> { where(status: 'pending') } - scope :success, -> { where(status: 'success') } - scope :failed, -> { where(status: 'failed') } - scope :canceled, -> { where(status: 'canceled') } - scope :skipped, -> { where(status: 'skipped') } - scope :manual, -> { where(status: 'manual') } - - scope :failed_but_allowed, -> do - where(allow_failure: true, status: [:failed, :canceled]) - end - - scope :exclude_ignored, -> do - where("allow_failure = ? OR status IN (?)", - false, %w[created pending running success skipped]) - end - - def self.status_sql - scope_relevant = latest.exclude_ignored - scope_warnings = latest.failed_but_allowed - - builds = scope_relevant.select('count(*)').to_sql - created = scope_relevant.created.select('count(*)').to_sql - success = scope_relevant.success.select('count(*)').to_sql - manual = scope_relevant.manual.select('count(*)').to_sql - pending = scope_relevant.pending.select('count(*)').to_sql - running = scope_relevant.running.select('count(*)').to_sql - skipped = scope_relevant.skipped.select('count(*)').to_sql - canceled = scope_relevant.canceled.select('count(*)').to_sql - warnings = scope_warnings.select('count(*) > 0').to_sql - - <<-SQL.strip_heredoc - (CASE - WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]} - WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]} - WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]} - WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]} - WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]} - WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]} - WHEN (#{created}) > 0 THEN #{STATUSES[:running]} - ELSE #{STATUSES[:failed]} - END) - SQL - end + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' end def up - disable_statement_timeout + index = 1 - status_sql = Build - .where('ci_builds.commit_id = ci_stages.pipeline_id') - .where('ci_builds.stage = ci_stages.name') - .status_sql + Stage.where(status: nil).in_batches(of: BATCH_SIZE) do |relation| + jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } + schedule = index * 5.minutes + index += 1 - update_column_in_batches(:ci_stages, :status, Arel.sql("(#{status_sql})")) do |table, query| - query.where(table[:status].eq(nil)) + BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) end end -- cgit v1.2.1