Reschedule commits_count background migrationjprovazn-commits-count-reschedule

We still have >100K unmigrated MergeRequestDiffs
which don't have commits_count set yet (see
https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/17567#note_61904891)

This migration re-schedules the original background migration.
To assure that records are not processed twice, records with
commits_count set are skipped.

Related to #41698 and !17567

5 files changed, 81 insertions, 2 deletions

diff --git a/db/migrate/20180309121820_reschedule_commits_count_for_merge_request_diff.rb b/db/migrate/20180309121820_reschedule_commits_count_for_merge_request_diff.rb
new file mode 100644
index 00000000000..990759104b0
--- /dev/null
+++ b/db/migrate/20180309121820_reschedule_commits_count_for_merge_request_diff.rb
@@ -0,0 +1,30 @@
+class RescheduleCommitsCountForMergeRequestDiff < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  MIGRATION = 'AddMergeRequestDiffCommitsCount'.freeze
+  BATCH_SIZE = 5000
+  DELAY_INTERVAL = 5.minutes.to_i
+
+  class MergeRequestDiff < ActiveRecord::Base
+    self.table_name = 'merge_request_diffs'
+
+    include ::EachBatch
+  end
+
+  disable_ddl_transaction!
+
+  def up
+    say 'Populating the MergeRequestDiff `commits_count` (reschedule)'
+
+    execute("SET statement_timeout TO '60s'") if Gitlab::Database.postgresql?
+
+    MergeRequestDiff.where(commits_count: nil).each_batch(of: BATCH_SIZE) do |relation, index|
+      start_id, end_id = relation.pluck('MIN(id), MAX(id)').first
+      delay = index * DELAY_INTERVAL
+
+      BackgroundMigrationWorker.perform_in(delay, MIGRATION, [start_id, end_id])
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 75a094bbbb6..970b1ad9948 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180308052825) do
+ActiveRecord::Schema.define(version: 20180309121820) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb
index 7bffffec94d..d5cf9e0d53a 100644
--- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb
+++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb
@@ -19,7 +19,7 @@ module Gitlab
             WHERE merge_request_diffs.id = merge_request_diff_commits.merge_request_diff_id
           )'.squish
 
-        MergeRequestDiff.where(id: start_id..stop_id).update_all(update)
+        MergeRequestDiff.where(id: start_id..stop_id).where(commits_count: nil).update_all(update)
       end
     end
   end
diff --git a/spec/lib/gitlab/background_migration/add_merge_request_diff_commits_count_spec.rb b/spec/lib/gitlab/background_migration/add_merge_request_diff_commits_count_spec.rb
index 21a791f5695..c43ed72038e 100644
--- a/spec/lib/gitlab/background_migration/add_merge_request_diff_commits_count_spec.rb
+++ b/spec/lib/gitlab/background_migration/add_merge_request_diff_commits_count_spec.rb
@@ -37,6 +37,18 @@ describe Gitlab::BackgroundMigration::AddMergeRequestDiffCommitsCount, :migratio
       expect(diff.reload.commits_count).to eq(0)
     end
 
+    it 'skips diffs that have commits_count already set' do
+      timestamp = 2.days.ago
+      diff = merge_request_diffs_table.create!(
+        merge_request_id: merge_request.id,
+        commits_count: 0,
+        updated_at: timestamp)
+
+      subject.perform(diff.id, diff.id)
+
+      expect(diff.reload.updated_at).to be_within(1.second).of(timestamp)
+    end
+
     it 'migrates multiple diffs to the correct values' do
       diffs = Array.new(3).map.with_index { |_, i| create_diff!(i, commits: 3) }
 
diff --git a/spec/migrations/reschedule_commits_count_for_merge_request_diff_spec.rb b/spec/migrations/reschedule_commits_count_for_merge_request_diff_spec.rb
new file mode 100644
index 00000000000..26489ef58bd
--- /dev/null
+++ b/spec/migrations/reschedule_commits_count_for_merge_request_diff_spec.rb
@@ -0,0 +1,37 @@
+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20180309121820_reschedule_commits_count_for_merge_request_diff')
+
+describe RescheduleCommitsCountForMergeRequestDiff, :migration, :sidekiq do
+  let(:merge_request_diffs) { table(:merge_request_diffs) }
+  let(:merge_requests) { table(:merge_requests) }
+  let(:projects) { table(:projects) }
+  let(:namespaces) { table(:namespaces) }
+
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 1)
+
+    namespaces.create!(id: 1, name: 'gitlab', path: 'gitlab')
+
+    projects.create!(id: 1, namespace_id: 1)
+
+    merge_requests.create!(id: 1, target_project_id: 1, source_project_id: 1, target_branch: 'feature', source_branch: 'master')
+
+    merge_request_diffs.create!(id: 1, merge_request_id: 1)
+    merge_request_diffs.create!(id: 2, merge_request_id: 1)
+    merge_request_diffs.create!(id: 3, merge_request_id: 1, commits_count: 0)
+    merge_request_diffs.create!(id: 4, merge_request_id: 1)
+  end
+
+  it 'correctly schedules background migrations' do
+    Sidekiq::Testing.fake! do
+      Timecop.freeze do
+        migrate!
+
+        expect(described_class::MIGRATION).to be_scheduled_delayed_migration(5.minutes, 1, 1)
+        expect(described_class::MIGRATION).to be_scheduled_delayed_migration(10.minutes, 2, 2)
+        expect(described_class::MIGRATION).to be_scheduled_delayed_migration(15.minutes, 4, 4)
+        expect(BackgroundMigrationWorker.jobs.size).to eq 3
+      end
+    end
+  end
+end