POC for zero downtime migrations without having to manually manage what migration goes into what releaserelease-migrations

8 files changed, 231 insertions, 12 deletions

diff --git a/app/models/concerns/ignorable_column.rb b/app/models/concerns/ignorable_column.rb
new file mode 100644
index 00000000000..eb9f3423e48
--- /dev/null
+++ b/app/models/concerns/ignorable_column.rb
@@ -0,0 +1,28 @@
+# Module that can be included into a model to make it easier to ignore database
+# columns.
+#
+# Example:
+#
+#     class User < ActiveRecord::Base
+#       include IgnorableColumn
+#
+#       ignore_column :updated_at
+#     end
+#
+module IgnorableColumn
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def columns
+      super.reject { |column| ignored_columns.include?(column.name) }
+    end
+
+    def ignored_columns
+      @ignored_columns ||= Set.new
+    end
+
+    def ignore_column(name)
+      ignored_columns << name.to_s
+    end
+  end
+end
diff --git a/app/models/concerns/renameable_column.rb b/app/models/concerns/renameable_column.rb
new file mode 100644
index 00000000000..3f905520fe9
--- /dev/null
+++ b/app/models/concerns/renameable_column.rb
@@ -0,0 +1,107 @@
+module RenameableColumn
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def rename_column(old_column, new_column, migrations:)
+      unless migration_exists?(migrations[:add_new])
+        raise "Cannot find migration '#{migrations[:add_new]}'"
+      end
+
+      unless migration_exists?(migrations[:migrate_data])
+        raise "Cannot find migration '#{migrations[:migrate_data]}'"
+      end
+
+      unless migration_exists?(migrations[:remove_old])
+        raise "Cannot find migration '#{migrations[:remove_old]}'"
+      end
+
+      if migration_ran?(migrations[:remove_old])
+        old_column_removed(old_column, new_column)
+      elsif migration_ran?(migrations[:migrate_data])
+        data_migrated_from_old_to_new_column(old_column, new_column)
+      elsif migration_ran?(migrations[:add_new])
+        new_column_added(old_column, new_column)
+      end
+    end
+
+    private
+
+    def new_column_added(old_column, new_column)
+      log_column_rename_status(old_column, new_column)
+      Rails.logger.info "The `#{new_column}` column has been added, but the data has not yet been migrated, and the `#{old_column}` column has not yet been removed."
+      log_column_usage_instructions(old_column, new_column)
+
+      before_save do
+        self[new_column] = self[old_column]
+
+        true
+      end
+
+      define_method "#{new_column}=" do |new_value|
+        raise "Use `#{self.class.name}##{old_column}=` until data is migrated from `#{old_column}` to `#{new_column}`"
+      end
+
+      define_singleton_method column_name_method(new_column) do
+        old_column
+      end
+    end
+
+    def data_migrated_from_old_to_new_column(old_column, new_column)
+      log_column_rename_status(old_column, new_column)
+      Rails.logger.info "The `#{new_column}` column has been added and the data has been migrated, but the `#{old_column}` column has not yet been removed."
+      log_column_usage_instructions(old_column, new_column)
+
+      # We read and write to and from `new_column`, but the code still says `old_column`
+      include IgnorableColumn
+
+      ignore_column old_column
+
+      alias_attribute old_column, new_column
+
+      define_singleton_method column_name_method(new_column) do
+        new_column
+      end
+    end
+
+    def old_column_removed(old_column, new_column)
+      warn "WARNING: `#{self.name}` column `#{old_column}` has been renamed to `#{new_column}`."
+      warn "All code should be updated to use `#{new_column}` where `#{old_column}` or the value of `#{self.name}.#{column_name_method(new_column)}` is currently used, and `rename_column #{old_column.inspect}, #{new_column.inspect}` should be removed from `#{self.name}`."
+
+      alias_attribute old_column, new_column
+
+      define_singleton_method column_name_method(new_column) do
+        new_column
+      end
+    end
+
+    def log_column_rename_status(old_column, new_column)
+      Rails.logger.info "`#{self.name}` column `#{old_column}` is in the process of being renamed to `#{new_column}`."
+    end
+
+    def log_column_usage_instructions(old_column, new_column)
+      Rails.logger.info "Until the rename is complete, all code should continue to read and write from `#{old_column}`, which may be the actual attribute or an ActiveRecord alias. However, plain SQL and Arel queries should use `#{self.name}.#{column_name_method(new_column)}`, which will return `#{old_column.inspect}` or `#{new_column.inspect}` based on the current state of the database."
+    end
+
+    def column_name_method(new_column)
+      "#{new_column}_column_name"
+    end
+
+    def all_migrations
+      @all_migrations ||= Set.new(ActiveRecord::Migrator.migrations(['db/release_migrations']).map(&:version))
+    end
+
+    def migration_exists?(name)
+      version = name.match(/^([0-9]+)/)[1].to_i
+      all_migrations.include?(version)
+    end
+
+    def ran_migrations
+      @ran_migrations ||= Set.new(ActiveRecord::Migrator.get_all_versions)
+    end
+
+    def migration_ran?(name)
+      version = name.match(/^([0-9]+)/)[1].to_i
+      ran_migrations.include?(version)
+    end
+  end
+end
diff --git a/app/models/user.rb b/app/models/user.rb
index 39c1281179b..a3bec974f1d 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -1,6 +1,13 @@
 require 'carrierwave/orm/activerecord'
 
 class User < ActiveRecord::Base
+  include RenameableColumn
+  rename_column :username, :handle, migrations: {
+    add_new:      '20170316184328_add_handle_column_to_users',
+    migrate_data: '20170316190016_migrate_users_username_to_handle',
+    remove_old:   '20170316190051_remove_username_column_from_users'
+  }
+
   extend Gitlab::ConfigHelper
 
   include Gitlab::ConfigHelper
diff --git a/config/initializers/0_post_deployment_migrations.rb b/config/initializers/0_post_deployment_migrations.rb
deleted file mode 100644
index 0068a03d214..00000000000
--- a/config/initializers/0_post_deployment_migrations.rb
+++ /dev/null
@@ -1,12 +0,0 @@
-# Post deployment migrations are included by default. This file must be loaded
-# before other initializers as Rails may otherwise memoize a list of migrations
-# excluding the post deployment migrations.
-unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS']
-  path = Rails.root.join('db', 'post_migrate').to_s
-
-  Rails.application.config.paths['db/migrate'] << path
-
-  # Rails memoizes migrations at certain points where it won't read the above
-  # path just yet. As such we must also update the following list of paths.
-  ActiveRecord::Migrator.migrations_paths << path
-end
diff --git a/config/initializers/3_extra_migrations.rb b/config/initializers/3_extra_migrations.rb
new file mode 100644
index 00000000000..e2d55349425
--- /dev/null
+++ b/config/initializers/3_extra_migrations.rb
@@ -0,0 +1,29 @@
+current_version = Gitlab::VersionInfo.parse(Gitlab::VERSION)
+
+past_version_paths = Dir.glob(Rails.root.join('db', 'release_migrations', '*').to_s).select do |path|
+  version_string = path.match(%r{db/release_migrations/((\d+)\.(\d+)\.(\d+))})[1]
+  version = Gitlab::VersionInfo.parse(version_string)
+  version <= current_version
+end
+
+migrate_paths = past_version_paths.map { |path| File.join(path, 'migrate') }
+
+Rails.application.config.paths['db/migrate'].concat(migrate_paths)
+
+# Rails memoizes migrations at certain points where it won't read the above
+# path just yet. As such we must also update the following list of paths.
+ActiveRecord::Migrator.migrations_paths.concat(migrate_paths)
+
+# Post deployment migrations are included by default. This file must be loaded
+# before other initializers as Rails may otherwise memoize a list of migrations
+# excluding the post deployment migrations.
+unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS']
+  post_migrate_paths = past_version_paths.map { |path| File.join(path, 'post_migrate') }
+  post_migrate_paths << Rails.root.join('db', 'post_migrate').to_s
+
+  Rails.application.config.paths['db/migrate'].concat(post_migrate_paths)
+
+  # Rails memoizes migrations at certain points where it won't read the above
+  # path just yet. As such we must also update the following list of paths.
+  ActiveRecord::Migrator.migrations_paths.concat(post_migrate_paths)
+end
diff --git a/db/release_migrations/9.0.0/migrate/20170316184328_add_handle_column_to_users.rb b/db/release_migrations/9.0.0/migrate/20170316184328_add_handle_column_to_users.rb
new file mode 100644
index 00000000000..cc4a7fee7d9
--- /dev/null
+++ b/db/release_migrations/9.0.0/migrate/20170316184328_add_handle_column_to_users.rb
@@ -0,0 +1,13 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddHandleColumnToUsers < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    add_column :users, :handle, :string
+  end
+end
diff --git a/db/release_migrations/9.0.0/post_migrate/20170316190016_migrate_users_username_to_handle.rb b/db/release_migrations/9.0.0/post_migrate/20170316190016_migrate_users_username_to_handle.rb
new file mode 100644
index 00000000000..965ce691b98
--- /dev/null
+++ b/db/release_migrations/9.0.0/post_migrate/20170316190016_migrate_users_username_to_handle.rb
@@ -0,0 +1,23 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class MigrateUsersUsernameToHandle < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    old_column = Arel::Table.new(:users)[:username]
+
+    # This will set users.handle to users.username in batches.
+    update_column_in_batches(:users, :handle, old_column)
+  end
+
+  def down
+    old_column = Arel::Table.new(:users)[:handle]
+
+    update_column_in_batches(:users, :username, old_column)
+  end
+end
diff --git a/db/release_migrations/9.0.1/post_migrate/20170316190051_remove_username_column_from_users.rb b/db/release_migrations/9.0.1/post_migrate/20170316190051_remove_username_column_from_users.rb
new file mode 100644
index 00000000000..0a6520f698a
--- /dev/null
+++ b/db/release_migrations/9.0.1/post_migrate/20170316190051_remove_username_column_from_users.rb
@@ -0,0 +1,24 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveUsernameColumnFromUsers < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    remove_column :users, :username
+  end
+
+  def down
+    add_column :users, :username, :string
+
+    # Populate the column with data again so we can safely revert the migration
+    # without losing any data.
+    old_column = Arel::Table.new(:users)[:handle]
+
+    update_column_in_batches(:users, :username, old_column)
+  end
+end