Finish migration to the new events setupevents-migration-cleanup

This finishes the procedure for migrating events from the old format
into the new format. Code no longer uses the old setup and the database
tables used during the migration process are swapped, with the old table
being dropped.

While the database migration can be reversed this will 1) take a lot of
time as data has to be coped around 2) won't restore data in the
"events.data" column as we have no way of restoring this.

Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/37241

5 files changed, 96 insertions, 28 deletions

diff --git a/db/migrate/20170830130119_steal_remaining_event_migration_jobs.rb b/db/migrate/20170830130119_steal_remaining_event_migration_jobs.rb
new file mode 100644
index 00000000000..0dfdc4ed261
--- /dev/null
+++ b/db/migrate/20170830130119_steal_remaining_event_migration_jobs.rb
@@ -0,0 +1,18 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class StealRemainingEventMigrationJobs < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    Gitlab::BackgroundMigration.steal('MigrateEventsToPushEventPayloads')
+  end
+
+  def down
+  end
+end
diff --git a/db/migrate/20170830131015_swap_event_migration_tables.rb b/db/migrate/20170830131015_swap_event_migration_tables.rb
new file mode 100644
index 00000000000..5128d1b2fe7
--- /dev/null
+++ b/db/migrate/20170830131015_swap_event_migration_tables.rb
@@ -0,0 +1,23 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class SwapEventMigrationTables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def up
+    rename_tables
+  end
+
+  def down
+    rename_tables
+  end
+
+  def rename_tables
+    rename_table :events, :events_old
+    rename_table :events_for_migration, :events
+    rename_table :events_old, :events_for_migration
+  end
+end
diff --git a/db/migrate/limits_to_mysql.rb b/db/migrate/limits_to_mysql.rb
index be3501c4c2e..5cd9f3198e3 100644
--- a/db/migrate/limits_to_mysql.rb
+++ b/db/migrate/limits_to_mysql.rb
@@ -7,6 +7,5 @@ class LimitsToMysql < ActiveRecord::Migration
     change_column :merge_request_diffs, :st_diffs, :text, limit: 2147483647
     change_column :snippets, :content, :text, limit: 2147483647
     change_column :notes, :st_diff, :text, limit: 2147483647
-    change_column :events, :data, :text, limit: 2147483647
   end
 end
diff --git a/db/post_migrate/20170830150306_drop_events_for_migration_table.rb b/db/post_migrate/20170830150306_drop_events_for_migration_table.rb
new file mode 100644
index 00000000000..763ee9a810d
--- /dev/null
+++ b/db/post_migrate/20170830150306_drop_events_for_migration_table.rb
@@ -0,0 +1,48 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class DropEventsForMigrationTable < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class Event < ActiveRecord::Base
+    include EachBatch
+  end
+
+  def up
+    transaction do
+      drop_table :events_for_migration
+    end
+  end
+
+  # rubocop: disable Migration/Datetime
+  def down
+    create_table :events_for_migration do |t|
+      t.string :target_type, index: true
+      t.integer :target_id, index: true
+      t.string :title
+      t.text :data
+      t.integer :project_id
+      t.datetime :created_at, index: true
+      t.datetime :updated_at
+      t.integer :action, index: true
+      t.integer :author_id, index: true
+
+      t.index [:project_id, :id]
+    end
+
+    Event.all.each_batch do |relation|
+      start_id, stop_id = relation.pluck('MIN(id), MAX(id)').first
+
+      execute <<-EOF.strip_heredoc
+      INSERT INTO events_for_migration (target_type, target_id, project_id, created_at, updated_at, action, author_id)
+      SELECT target_type, target_id, project_id, created_at, updated_at, action, author_id
+      FROM events
+      WHERE id BETWEEN #{start_id} AND #{stop_id}
+      EOF
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f980667a38f..5a0729ccfea 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -531,25 +531,6 @@ ActiveRecord::Schema.define(version: 20170901071411) do
   add_index "environments", ["project_id", "slug"], name: "index_environments_on_project_id_and_slug", unique: true, using: :btree
 
   create_table "events", force: :cascade do |t|
-    t.string "target_type"
-    t.integer "target_id"
-    t.string "title"
-    t.text "data"
-    t.integer "project_id"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.integer "action"
-    t.integer "author_id"
-  end
-
-  add_index "events", ["action"], name: "index_events_on_action", using: :btree
-  add_index "events", ["author_id"], name: "index_events_on_author_id", using: :btree
-  add_index "events", ["created_at"], name: "index_events_on_created_at", using: :btree
-  add_index "events", ["project_id", "id"], name: "index_events_on_project_id_and_id", using: :btree
-  add_index "events", ["target_id"], name: "index_events_on_target_id", using: :btree
-  add_index "events", ["target_type"], name: "index_events_on_target_type", using: :btree
-
-  create_table "events_for_migration", force: :cascade do |t|
     t.integer "project_id"
     t.integer "author_id", null: false
     t.integer "target_id"
@@ -559,10 +540,10 @@ ActiveRecord::Schema.define(version: 20170901071411) do
     t.string "target_type"
   end
 
-  add_index "events_for_migration", ["action"], name: "index_events_for_migration_on_action", using: :btree
-  add_index "events_for_migration", ["author_id"], name: "index_events_for_migration_on_author_id", using: :btree
-  add_index "events_for_migration", ["project_id", "id"], name: "index_events_for_migration_on_project_id_and_id", using: :btree
-  add_index "events_for_migration", ["target_type", "target_id"], name: "index_events_for_migration_on_target_type_and_target_id", using: :btree
+  add_index "events", ["action"], name: "index_events_on_action", using: :btree
+  add_index "events", ["author_id"], name: "index_events_on_author_id", using: :btree
+  add_index "events", ["project_id", "id"], name: "index_events_on_project_id_and_id", using: :btree
+  add_index "events", ["target_type", "target_id"], name: "index_events_on_target_type_and_target_id", using: :btree
 
   create_table "feature_gates", force: :cascade do |t|
     t.string "feature_key", null: false
@@ -1697,9 +1678,8 @@ ActiveRecord::Schema.define(version: 20170901071411) do
   add_foreign_key "deploy_keys_projects", "projects", name: "fk_58a901ca7e", on_delete: :cascade
   add_foreign_key "deployments", "projects", name: "fk_b9a3851b82", on_delete: :cascade
   add_foreign_key "environments", "projects", name: "fk_d1c8c1da6a", on_delete: :cascade
-  add_foreign_key "events", "projects", name: "fk_0434b48643", on_delete: :cascade
-  add_foreign_key "events_for_migration", "projects", on_delete: :cascade
-  add_foreign_key "events_for_migration", "users", column: "author_id", name: "fk_edfd187b6f", on_delete: :cascade
+  add_foreign_key "events", "projects", on_delete: :cascade
+  add_foreign_key "events", "users", column: "author_id", name: "fk_edfd187b6f", on_delete: :cascade
   add_foreign_key "forked_project_links", "projects", column: "forked_to_project_id", name: "fk_434510edb0", on_delete: :cascade
   add_foreign_key "gpg_keys", "users", on_delete: :cascade
   add_foreign_key "gpg_signatures", "gpg_keys", on_delete: :nullify
@@ -1743,7 +1723,7 @@ ActiveRecord::Schema.define(version: 20170901071411) do
   add_foreign_key "protected_tag_create_access_levels", "protected_tags", name: "fk_f7dfda8c51", on_delete: :cascade
   add_foreign_key "protected_tag_create_access_levels", "users"
   add_foreign_key "protected_tags", "projects", name: "fk_8e4af87648", on_delete: :cascade
-  add_foreign_key "push_event_payloads", "events_for_migration", column: "event_id", name: "fk_36c74129da", on_delete: :cascade
+  add_foreign_key "push_event_payloads", "events", name: "fk_36c74129da", on_delete: :cascade
   add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade
   add_foreign_key "services", "projects", name: "fk_71cce407f9", on_delete: :cascade
   add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade