Merge branch 'master' into backstage/gb/build-stages-catch-up-migration

* master: (1480 commits)

Conflicts:
	db/schema.rb

12 files changed, 605 insertions, 46 deletions

diff --git a/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb b/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb
index da0fcda87a6..17ad7de065d 100644
--- a/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb
+++ b/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb
@@ -31,6 +31,7 @@ class RenameUsersWithRenamedNamespace < ActiveRecord::Migration
       predicate = namespaces[:owner_id].eq(users[:id])
                     .and(namespaces[:type].eq(nil))
                     .and(users[:username].matches(path))
+
       update_sql = if Gitlab::Database.postgresql?
                      "UPDATE users SET username = namespaces.path "\
                      "FROM namespaces WHERE #{predicate.to_sql}"
diff --git a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb
new file mode 100644
index 00000000000..11b581e4b57
--- /dev/null
+++ b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb
@@ -0,0 +1,151 @@
+class MigrateKubernetesServiceToNewClustersArchitectures < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME = 'KubernetesService'.freeze
+
+  disable_ddl_transaction!
+
+  class Project < ActiveRecord::Base
+    self.table_name = 'projects'
+
+    has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject'
+    has_many :clusters, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+    has_many :services, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service'
+    has_one :kubernetes_service, -> { where(category: 'deployment', type: 'KubernetesService') }, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service', inverse_of: :project, foreign_key: :project_id
+  end
+
+  class Cluster < ActiveRecord::Base
+    self.table_name = 'clusters'
+
+    has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject'
+    has_many :projects, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project'
+    has_one :platform_kubernetes, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::PlatformsKubernetes'
+
+    accepts_nested_attributes_for :platform_kubernetes
+
+    enum platform_type: {
+      kubernetes: 1
+    }
+
+    enum provider_type: {
+      user: 0,
+      gcp: 1
+    }
+  end
+
+  class ClustersProject < ActiveRecord::Base
+    self.table_name = 'cluster_projects'
+
+    belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+    belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project'
+  end
+
+  class PlatformsKubernetes < ActiveRecord::Base
+    self.table_name = 'cluster_platforms_kubernetes'
+
+    belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+
+    attr_encrypted :token,
+      mode: :per_attribute_iv,
+      key: Gitlab::Application.secrets.db_key_base,
+      algorithm: 'aes-256-cbc'
+  end
+
+  class Service < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'services'
+    self.inheritance_column = :_type_disabled # Disable STI, otherwise KubernetesModel will be looked up
+
+    belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project', foreign_key: :project_id
+
+    scope :unmanaged_kubernetes_service, -> do
+      joins('LEFT JOIN projects ON projects.id = services.project_id')
+      .joins('LEFT JOIN cluster_projects ON cluster_projects.project_id = projects.id')
+      .joins('LEFT JOIN cluster_platforms_kubernetes ON cluster_platforms_kubernetes.cluster_id = cluster_projects.cluster_id')
+      .where(category: 'deployment', type: 'KubernetesService', template: false)
+      .where("services.properties LIKE '%api_url%'")
+      .where("(services.properties NOT LIKE CONCAT('%', cluster_platforms_kubernetes.api_url, '%')) OR cluster_platforms_kubernetes.api_url IS NULL")
+      .group(:id)
+      .order(id: :asc)
+    end
+
+    scope :kubernetes_service_without_template, -> do
+      where(category: 'deployment', type: 'KubernetesService', template: false)
+    end
+
+    def api_url
+      parsed_properties['api_url']
+    end
+
+    def ca_pem
+      parsed_properties['ca_pem']
+    end
+
+    def namespace
+      parsed_properties['namespace']
+    end
+
+    def token
+      parsed_properties['token']
+    end
+
+    private
+
+    def parsed_properties
+      @parsed_properties ||= JSON.parse(self.properties)
+    end
+  end
+
+  def find_dedicated_environement_scope(project)
+    environment_scopes = project.clusters.map(&:environment_scope)
+
+    return '*' if environment_scopes.exclude?('*') # KubernetesService should be added as a default cluster (environment_scope: '*') at first place
+    return 'migrated/*' if environment_scopes.exclude?('migrated/*') # If it's conflicted, the KubernetesService added as a migrated cluster
+
+    unique_iid = 0
+
+    # If it's still conflicted, finding an unique environment scope incrementaly
+    loop do
+      candidate = "migrated#{unique_iid}/*"
+      return candidate if environment_scopes.exclude?(candidate)
+
+      unique_iid += 1
+    end
+  end
+
+  def up
+    ActiveRecord::Base.transaction do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service
+        .unmanaged_kubernetes_service.find_each(batch_size: 1) do |kubernetes_service|
+        MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create(
+          enabled: kubernetes_service.active,
+          user_id: nil, # KubernetesService doesn't have
+          name: DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME,
+          provider_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.provider_types[:user],
+          platform_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.platform_types[:kubernetes],
+          projects: [kubernetes_service.project],
+          environment_scope: find_dedicated_environement_scope(kubernetes_service.project),
+          platform_kubernetes_attributes: {
+            api_url: kubernetes_service.api_url,
+            ca_cert: kubernetes_service.ca_pem,
+            namespace: kubernetes_service.namespace,
+            username: nil, # KubernetesService doesn't have
+            encrypted_password: nil, # KubernetesService doesn't have
+            encrypted_password_iv: nil, # KubernetesService doesn't have
+            token: kubernetes_service.token # encrypted_token and encrypted_token_iv
+          } )
+      end
+    end
+
+    MigrateKubernetesServiceToNewClustersArchitectures::Service
+      .kubernetes_service_without_template.each_batch(of: 100) do |kubernetes_service|
+      kubernetes_service.update_all(active: false)
+    end
+  end
+
+  def down
+    # noop
+  end
+end
diff --git a/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb b/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb
index 05430efe1f6..26f917d5a1e 100644
--- a/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb
+++ b/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb
@@ -3,22 +3,8 @@ class RescheduleForkNetworkCreation < ActiveRecord::Migration
 
   DOWNTIME = false
 
-  MIGRATION = 'PopulateForkNetworksRange'.freeze
-  BATCH_SIZE = 100
-  DELAY_INTERVAL = 15.seconds
-
-  disable_ddl_transaction!
-
-  class ForkedProjectLink < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'forked_project_links'
-  end
-
   def up
-    say 'Populating the `fork_networks` based on existing `forked_project_links`'
-
-    queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+    say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller'
   end
 
   def down
diff --git a/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb
index 547cc68e10e..fce1829c982 100644
--- a/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb
+++ b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb
@@ -15,8 +15,6 @@ class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migratio
   end
 
   def up
-    merge_requests = MergeRequest.where("id IN (#{updatable_merge_requests_union_sql})").reorder(:id)
-
     say 'Scheduling `PopulateMergeRequestMetricsWithEventsData` jobs'
     # It will update around 4_000_000 records in batches of 10_000 merge
     # requests (running between 10 minutes) and should take around 66 hours to complete.
@@ -25,7 +23,7 @@ class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migratio
     #
     # More information about the updates in `PopulateMergeRequestMetricsWithEventsData` class.
     #
-    merge_requests.each_batch(of: BATCH_SIZE) do |relation, index|
+    MergeRequest.all.each_batch(of: BATCH_SIZE) do |relation, index|
       range = relation.pluck('MIN(id)', 'MAX(id)').first
 
       BackgroundMigrationWorker.perform_in(index * 10.minutes, MIGRATION, range)
@@ -37,32 +35,4 @@ class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migratio
     execute "update merge_request_metrics set latest_closed_by_id = null"
     execute "update merge_request_metrics set merged_by_id = null"
   end
-
-  private
-
-  # On staging:
-  # Planning time: 0.682 ms
-  # Execution time: 22033.158 ms
-  #
-  def updatable_merge_requests_union_sql
-    metrics_not_exists_clause =
-      'NOT EXISTS (SELECT 1 FROM merge_request_metrics WHERE merge_request_metrics.merge_request_id = merge_requests.id)'
-
-    without_metrics_data = <<-SQL.strip_heredoc
-      merge_request_metrics.merged_by_id IS NULL OR
-      merge_request_metrics.latest_closed_by_id IS NULL OR
-      merge_request_metrics.latest_closed_at IS NULL
-    SQL
-
-    mrs_without_metrics_record = MergeRequest
-      .where(metrics_not_exists_clause)
-      .select(:id)
-
-    mrs_without_events_data = MergeRequest
-      .joins('INNER JOIN merge_request_metrics ON merge_requests.id = merge_request_metrics.merge_request_id')
-      .where(without_metrics_data)
-      .select(:id)
-
-    Gitlab::SQL::Union.new([mrs_without_metrics_record, mrs_without_events_data]).to_sql
-  end
 end
diff --git a/db/post_migrate/20171207150300_remove_project_labels_group_id_copy.rb b/db/post_migrate/20171207150300_remove_project_labels_group_id_copy.rb
new file mode 100644
index 00000000000..2f339172eeb
--- /dev/null
+++ b/db/post_migrate/20171207150300_remove_project_labels_group_id_copy.rb
@@ -0,0 +1,21 @@
+# Copy of 20180202111106 - this one should run before 20171207150343 to fix issues related to
+# the removal of groups with labels.
+
+class RemoveProjectLabelsGroupIdCopy < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    # rubocop:disable Migration/UpdateColumnInBatches
+    update_column_in_batches(:labels, :group_id, nil) do |table, query|
+      query.where(table[:type].eq('ProjectLabel').and(table[:group_id].not_eq(nil)))
+    end
+    # rubocop:enable Migration/UpdateColumnInBatches
+  end
+
+  def down
+  end
+end
diff --git a/db/post_migrate/20171207150343_remove_soft_removed_objects.rb b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb
new file mode 100644
index 00000000000..3e2dedfdd6a
--- /dev/null
+++ b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb
@@ -0,0 +1,208 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveSoftRemovedObjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  module SoftRemoved
+    extend ActiveSupport::Concern
+
+    included do
+      scope :soft_removed, -> { where('deleted_at IS NOT NULL') }
+    end
+  end
+
+  class User < ActiveRecord::Base
+    self.table_name = 'users'
+
+    include EachBatch
+  end
+
+  class Issue < ActiveRecord::Base
+    self.table_name = 'issues'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class MergeRequest < ActiveRecord::Base
+    self.table_name = 'merge_requests'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class Namespace < ActiveRecord::Base
+    self.table_name = 'namespaces'
+
+    include EachBatch
+    include SoftRemoved
+
+    scope :soft_removed_personal, -> { soft_removed.where(type: nil) }
+    scope :soft_removed_group, -> { soft_removed.where(type: 'Group') }
+  end
+
+  class Route < ActiveRecord::Base
+    self.table_name = 'routes'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class Project < ActiveRecord::Base
+    self.table_name = 'projects'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class CiPipelineSchedule < ActiveRecord::Base
+    self.table_name = 'ci_pipeline_schedules'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class CiTrigger < ActiveRecord::Base
+    self.table_name = 'ci_triggers'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  MODELS = [Issue, MergeRequest, CiPipelineSchedule, CiTrigger].freeze
+
+  def up
+    disable_statement_timeout
+
+    remove_personal_routes
+    remove_personal_namespaces
+    remove_group_namespaces
+    remove_simple_soft_removed_rows
+  end
+
+  def down
+    # The data removed by this migration can't be restored in an automated way.
+  end
+
+  def remove_simple_soft_removed_rows
+    create_temporary_indexes
+
+    MODELS.each do |model|
+      say_with_time("Removing soft removed rows from #{model.table_name}") do
+        model.soft_removed.each_batch do |batch, index|
+          batch.delete_all
+        end
+      end
+    end
+  ensure
+    remove_temporary_indexes
+  end
+
+  def create_temporary_indexes
+    MODELS.each do |model|
+      index_name = temporary_index_name_for(model)
+
+      # Without this index the removal process can take a very long time. For
+      # example, getting the next ID of a batch for the `issues` table in
+      # staging would take between 15 and 20 seconds.
+      next if temporary_index_exists?(model)
+
+      say_with_time("Creating temporary index #{index_name}") do
+        add_concurrent_index(
+          model.table_name,
+          [:deleted_at, :id],
+          name: index_name,
+          where: 'deleted_at IS NOT NULL'
+        )
+      end
+    end
+  end
+
+  def remove_temporary_indexes
+    MODELS.each do |model|
+      index_name = temporary_index_name_for(model)
+
+      next unless temporary_index_exists?(model)
+
+      say_with_time("Removing temporary index #{index_name}") do
+        remove_concurrent_index_by_name(model.table_name, index_name)
+      end
+    end
+  end
+
+  def temporary_index_name_for(model)
+    "index_on_#{model.table_name}_tmp"
+  end
+
+  def temporary_index_exists?(model)
+    index_name = temporary_index_name_for(model)
+
+    index_exists?(model.table_name, [:deleted_at, :id], name: index_name)
+  end
+
+  def remove_personal_namespaces
+    # Some personal namespaces are left behind in case of GitLab.com. In these
+    # cases the associated data such as the projects and users has already been
+    # removed.
+    Namespace.soft_removed_personal.each_batch do |batch|
+      batch.delete_all
+    end
+  end
+
+  def remove_group_namespaces
+    admin_id = id_for_admin_user
+
+    unless admin_id
+      say 'Not scheduling soft removed groups for removal as no admin user ' \
+        'could be found. You will need to remove any such groups manually.'
+
+      return
+    end
+
+    # Left over groups can't be easily removed because we may also need to
+    # remove memberships, repositories, and other associated data. As a result
+    # we'll just schedule a Sidekiq job to remove these.
+    #
+    # As of January 5th, 2018 there are 36 groups that will be removed using
+    # this code.
+    Namespace.select(:id).soft_removed_group.each_batch(of: 10) do |batch, index|
+      batch.each do |ns|
+        schedule_group_removal(index * 5.minutes, ns.id, admin_id)
+      end
+    end
+  end
+
+  def schedule_group_removal(delay, group_id, user_id)
+    if migrate_inline?
+      GroupDestroyWorker.new.perform(group_id, user_id)
+    else
+      GroupDestroyWorker.perform_in(delay, group_id, user_id)
+    end
+  end
+
+  def remove_personal_routes
+    namespaces = Namespace.select(1)
+      .soft_removed
+      .where('namespaces.type IS NULL')
+      .where('routes.source_type = ?', 'Namespace')
+      .where('routes.source_id = namespaces.id')
+
+    Route.where('EXISTS (?)', namespaces).each_batch do |batch|
+      batch.delete_all
+    end
+  end
+
+  def id_for_admin_user
+    User.where(admin: true).limit(1).pluck(:id).first
+  end
+
+  def migrate_inline?
+    Rails.env.test? || Rails.env.development?
+  end
+end
diff --git a/db/post_migrate/20171207150344_remove_deleted_at_columns.rb b/db/post_migrate/20171207150344_remove_deleted_at_columns.rb
new file mode 100644
index 00000000000..154d7a1b926
--- /dev/null
+++ b/db/post_migrate/20171207150344_remove_deleted_at_columns.rb
@@ -0,0 +1,31 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveDeletedAtColumns < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  TABLES = %i[issues merge_requests namespaces ci_pipeline_schedules ci_triggers].freeze
+  COLUMN = :deleted_at
+
+  def up
+    TABLES.each do |table|
+      remove_column(table, COLUMN) if column_exists?(table, COLUMN)
+    end
+  end
+
+  def down
+    TABLES.each do |table|
+      unless column_exists?(table, COLUMN)
+        add_column(table, COLUMN, :datetime_with_timezone)
+      end
+
+      unless index_exists?(table, COLUMN)
+        add_concurrent_index(table, COLUMN)
+      end
+    end
+  end
+end
diff --git a/db/post_migrate/20171215121205_post_populate_can_push_from_deploy_keys_projects.rb b/db/post_migrate/20171215121205_post_populate_can_push_from_deploy_keys_projects.rb
new file mode 100644
index 00000000000..3a5850df3db
--- /dev/null
+++ b/db/post_migrate/20171215121205_post_populate_can_push_from_deploy_keys_projects.rb
@@ -0,0 +1,63 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class PostPopulateCanPushFromDeployKeysProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DATABASE_NAME = Gitlab::Database.database_name
+
+  disable_ddl_transaction!
+
+  class DeploysKeyProject < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'deploy_keys_projects'
+  end
+
+  def up
+    DeploysKeyProject.each_batch(of: 10_000) do |batch|
+      start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
+
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET deploy_keys_projects.can_push = #{DATABASE_NAME}.keys.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects
+          SET can_push = keys.can_push
+          FROM keys
+          WHERE deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
+    end
+  end
+
+  def down
+    DeploysKeyProject.each_batch(of: 10_000) do |batch|
+      start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
+
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET #{DATABASE_NAME}.keys.can_push = deploy_keys_projects.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE keys
+          SET can_push = deploy_keys_projects.can_push
+          FROM deploy_keys_projects
+          WHERE deploy_keys_projects.deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
+    end
+  end
+end
diff --git a/db/post_migrate/20171215121259_remove_can_push_from_keys.rb b/db/post_migrate/20171215121259_remove_can_push_from_keys.rb
new file mode 100644
index 00000000000..0599811d986
--- /dev/null
+++ b/db/post_migrate/20171215121259_remove_can_push_from_keys.rb
@@ -0,0 +1,17 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveCanPushFromKeys < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  disable_ddl_transaction!
+
+  def up
+    remove_column :keys, :can_push
+  end
+
+  def down
+    add_column_with_default :keys, :can_push, :boolean, default: false, allow_null: false
+  end
+end
diff --git a/db/post_migrate/20180119121225_remove_redundant_pipeline_stages.rb b/db/post_migrate/20180119121225_remove_redundant_pipeline_stages.rb
new file mode 100644
index 00000000000..61ea85eb2a7
--- /dev/null
+++ b/db/post_migrate/20180119121225_remove_redundant_pipeline_stages.rb
@@ -0,0 +1,66 @@
+class RemoveRedundantPipelineStages < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up(attempts: 100)
+    remove_redundant_pipeline_stages!
+    remove_outdated_index!
+    add_unique_index!
+  rescue ActiveRecord::RecordNotUnique
+    retry if (attempts -= 1) > 0
+
+    raise StandardError, <<~EOS
+      Failed to add an unique index to ci_stages, despite retrying the
+      migration 100 times.
+
+      See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/16580.
+    EOS
+  end
+
+  def down
+    remove_concurrent_index :ci_stages, [:pipeline_id, :name], unique: true
+    add_concurrent_index :ci_stages, [:pipeline_id, :name]
+  end
+
+  private
+
+  def remove_outdated_index!
+    return unless index_exists?(:ci_stages, [:pipeline_id, :name])
+
+    remove_concurrent_index :ci_stages, [:pipeline_id, :name]
+  end
+
+  def add_unique_index!
+    add_concurrent_index :ci_stages, [:pipeline_id, :name], unique: true
+  end
+
+  def remove_redundant_pipeline_stages!
+    disable_statement_timeout
+
+    redundant_stages_ids = <<~SQL
+      SELECT id FROM ci_stages WHERE (pipeline_id, name) IN (
+        SELECT pipeline_id, name FROM ci_stages
+          GROUP BY pipeline_id, name HAVING COUNT(*) > 1
+      )
+    SQL
+
+    execute <<~SQL
+      UPDATE ci_builds SET stage_id = NULL WHERE stage_id IN (#{redundant_stages_ids})
+    SQL
+
+    if Gitlab::Database.postgresql?
+      execute <<~SQL
+        DELETE FROM ci_stages WHERE id IN (#{redundant_stages_ids})
+      SQL
+    else # We can't modify a table we are selecting from on MySQL
+      execute <<~SQL
+        DELETE a FROM ci_stages AS a, ci_stages AS b
+          WHERE a.pipeline_id = b.pipeline_id AND a.name = b.name
+            AND a.id <> b.id
+      SQL
+    end
+  end
+end
diff --git a/db/post_migrate/20180202111106_remove_project_labels_group_id.rb b/db/post_migrate/20180202111106_remove_project_labels_group_id.rb
new file mode 100644
index 00000000000..db7fd0d167d
--- /dev/null
+++ b/db/post_migrate/20180202111106_remove_project_labels_group_id.rb
@@ -0,0 +1,19 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveProjectLabelsGroupId < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    update_column_in_batches(:labels, :group_id, nil) do |table, query|
+      query.where(table[:type].eq('ProjectLabel').and(table[:group_id].not_eq(nil)))
+    end
+  end
+
+  def down
+  end
+end
diff --git a/db/post_migrate/20180204200836_change_author_id_to_not_null_in_todos.rb b/db/post_migrate/20180204200836_change_author_id_to_not_null_in_todos.rb
new file mode 100644
index 00000000000..92c32feebf7
--- /dev/null
+++ b/db/post_migrate/20180204200836_change_author_id_to_not_null_in_todos.rb
@@ -0,0 +1,26 @@
+class ChangeAuthorIdToNotNullInTodos < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  class Todo < ActiveRecord::Base
+    self.table_name = 'todos'
+    include EachBatch
+  end
+
+  BATCH_SIZE = 1000
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    Todo.where(author_id: nil).each_batch(of: BATCH_SIZE) do |batch|
+      batch.delete_all
+    end
+
+    change_column_null :todos, :author_id, false
+  end
+
+  def down
+    change_column_null :todos, :author_id, true
+  end
+end