summaryrefslogtreecommitdiff
path: root/db
diff options
context:
space:
mode:
authorFelipe Artur <fcardozo@gitlab.com>2019-05-06 19:45:17 +0000
committerSean McGivern <sean@gitlab.com>2019-05-06 19:45:17 +0000
commitc40bad741f963f0b3bf3868d485e419dbbf7b37c (patch)
treee41ea99f73a250154b8923eb2cca0ca1ce57ad61 /db
parentadc83d25a7ebf636941f72881a8828e6ebb5ad64 (diff)
downloadgitlab-ce-c40bad741f963f0b3bf3868d485e419dbbf7b37c.tar.gz
Fix issuables state_id nil when importing projects from GitHub
Issues and merge requests imported from GitHub are having state_id set to null. This fixes the GitHub project importer and schedule migrations to fix state_id.
Diffstat (limited to 'db')
-rw-r--r--db/migrate/20190506135337_add_temporary_indexes_to_state_id.rb34
-rw-r--r--db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb63
-rw-r--r--db/schema.rb2
3 files changed, 98 insertions, 1 deletions
diff --git a/db/migrate/20190506135337_add_temporary_indexes_to_state_id.rb b/db/migrate/20190506135337_add_temporary_indexes_to_state_id.rb
new file mode 100644
index 00000000000..8e9838e1afb
--- /dev/null
+++ b/db/migrate/20190506135337_add_temporary_indexes_to_state_id.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# This migration adds temporary indexes to state_id column of issues
+# and merge_requests tables. It will be used only to peform the scheduling
+# for populating state_id in a post migrate and will be removed after it.
+# Check: ScheduleSyncIssuablesStateIdWhereNil.
+
+class AddTemporaryIndexesToStateId < ActiveRecord::Migration[5.1]
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def up
+ %w(issues merge_requests).each do |table|
+ add_concurrent_index(
+ table,
+ 'id',
+ name: index_name_for(table),
+ where: "state_id IS NULL"
+ )
+ end
+ end
+
+ def down
+ remove_concurrent_index_by_name(:issues, index_name_for("issues"))
+ remove_concurrent_index_by_name(:merge_requests, index_name_for("merge_requests"))
+ end
+
+ def index_name_for(table)
+ "idx_on_#{table}_where_state_id_is_null"
+ end
+end
diff --git a/db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb b/db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb
new file mode 100644
index 00000000000..4c31b5968ff
--- /dev/null
+++ b/db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+class ScheduleSyncIssuablesStateIdWhereNil < ActiveRecord::Migration[5.1]
+ # Issues and MergeRequests imported by GitHub are being created with
+ # state_id = null, this fixes them.
+ #
+ # Part of a bigger plan: https://gitlab.com/gitlab-org/gitlab-ce/issues/51789
+
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ # 2019-05-02 gitlab.com issuable numbers
+ # issues with state_id nil: ~40000
+ # merge requests with state_id nil: ~200000
+ #
+ # Using 5000 as batch size and 120 seconds interval will create:
+ # ~8 jobs for issues - taking ~16 minutes
+ # ~40 jobs for merge requests - taking ~1.34 hours
+ #
+ BATCH_SIZE = 5000
+ DELAY_INTERVAL = 120.seconds.to_i
+ ISSUES_MIGRATION = 'SyncIssuesStateId'.freeze
+ MERGE_REQUESTS_MIGRATION = 'SyncMergeRequestsStateId'.freeze
+
+ disable_ddl_transaction!
+
+ class Issue < ActiveRecord::Base
+ include EachBatch
+
+ self.table_name = 'issues'
+ end
+
+ class MergeRequest < ActiveRecord::Base
+ include EachBatch
+
+ self.table_name = 'merge_requests'
+ end
+
+ def up
+ queue_background_migration_jobs_by_range_at_intervals(
+ Issue.where(state_id: nil),
+ ISSUES_MIGRATION,
+ DELAY_INTERVAL,
+ batch_size: BATCH_SIZE
+ )
+
+ queue_background_migration_jobs_by_range_at_intervals(
+ MergeRequest.where(state_id: nil),
+ MERGE_REQUESTS_MIGRATION,
+ DELAY_INTERVAL,
+ batch_size: BATCH_SIZE
+ )
+
+ # Remove temporary indexes added on "AddTemporaryIndexesToStateId"
+ remove_concurrent_index_by_name(:issues, "idx_on_issues_where_state_id_is_null")
+ remove_concurrent_index_by_name(:merge_requests, "idx_on_merge_requests_where_state_id_is_null")
+ end
+
+ def down
+ # No op
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1bcd22dc81c..deaf406fe3d 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20190426180107) do
+ActiveRecord::Schema.define(version: 20190506135400) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"