Merge branch 'master' into backstage/gb/build-stages-catch-up-migration

* master: (1480 commits)

Conflicts:
	db/schema.rb

37 files changed, 1165 insertions, 78 deletions

diff --git a/db/migrate/20160301174731_add_fingerprint_index.rb b/db/migrate/20160301174731_add_fingerprint_index.rb
new file mode 100644
index 00000000000..f2c3d1ba1ea
--- /dev/null
+++ b/db/migrate/20160301174731_add_fingerprint_index.rb
@@ -0,0 +1,17 @@
+# rubocop:disable all
+class AddFingerprintIndex < ActiveRecord::Migration
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  # https://gitlab.com/gitlab-org/gitlab-ee/issues/764
+  def change
+    args = [:keys, :fingerprint]
+
+    if Gitlab::Database.postgresql?
+      args << { algorithm: :concurrently }
+    end
+
+    add_index(*args) unless index_exists?(:keys, :fingerprint)
+  end
+end
diff --git a/db/migrate/20170425112128_create_pipeline_schedules_table.rb b/db/migrate/20170425112128_create_pipeline_schedules_table.rb
index 57df47f5f42..4f9c56a1ad8 100644
--- a/db/migrate/20170425112128_create_pipeline_schedules_table.rb
+++ b/db/migrate/20170425112128_create_pipeline_schedules_table.rb
@@ -17,7 +17,7 @@ class CreatePipelineSchedulesTable < ActiveRecord::Migration
       t.boolean :active, default: true
       t.datetime :deleted_at
 
-      t.timestamps
+      t.timestamps null: true
     end
 
     add_index(:ci_pipeline_schedules, :project_id)
diff --git a/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb b/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb
new file mode 100644
index 00000000000..1d86a531eb3
--- /dev/null
+++ b/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb
@@ -0,0 +1,19 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddAuthorizedKeysEnabledToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :application_settings, :authorized_keys_enabled, :boolean, default: true, allow_null: false
+  end
+
+  def down
+    remove_column :application_settings, :authorized_keys_enabled
+  end
+end
diff --git a/db/migrate/20170827123848_add_index_on_merge_request_diff_commit_sha.rb b/db/migrate/20170827123848_add_index_on_merge_request_diff_commit_sha.rb
new file mode 100644
index 00000000000..1b360b231a8
--- /dev/null
+++ b/db/migrate/20170827123848_add_index_on_merge_request_diff_commit_sha.rb
@@ -0,0 +1,17 @@
+# rubocop:disable RemoveIndex
+
+class AddIndexOnMergeRequestDiffCommitSha < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :merge_request_diff_commits, :sha, length: Gitlab::Database.mysql? ? 20 : nil
+  end
+
+  def down
+    remove_index :merge_request_diff_commits, :sha if index_exists? :merge_request_diff_commits, :sha
+  end
+end
diff --git a/db/migrate/20170928124105_create_fork_networks.rb b/db/migrate/20170928124105_create_fork_networks.rb
index ca906b953a3..89e5b871967 100644
--- a/db/migrate/20170928124105_create_fork_networks.rb
+++ b/db/migrate/20170928124105_create_fork_networks.rb
@@ -23,6 +23,7 @@ class CreateForkNetworks < ActiveRecord::Migration
     if foreign_keys_for(:fork_networks, :root_project_id).any?
       remove_foreign_key :fork_networks, column: :root_project_id
     end
+
     drop_table :fork_networks
   end
 end
diff --git a/db/migrate/20170928133643_create_fork_network_members.rb b/db/migrate/20170928133643_create_fork_network_members.rb
index 836f023efdc..8c7d9ba859a 100644
--- a/db/migrate/20170928133643_create_fork_network_members.rb
+++ b/db/migrate/20170928133643_create_fork_network_members.rb
@@ -21,6 +21,7 @@ class CreateForkNetworkMembers < ActiveRecord::Migration
     if foreign_keys_for(:fork_network_members, :forked_from_project_id).any?
       remove_foreign_key :fork_network_members, column: :forked_from_project_id
     end
+
     drop_table :fork_network_members
   end
 end
diff --git a/db/migrate/20170929131201_populate_fork_networks.rb b/db/migrate/20170929131201_populate_fork_networks.rb
index 1214962770f..ddbf27e1852 100644
--- a/db/migrate/20170929131201_populate_fork_networks.rb
+++ b/db/migrate/20170929131201_populate_fork_networks.rb
@@ -6,22 +6,8 @@ class PopulateForkNetworks < ActiveRecord::Migration
 
   DOWNTIME = false
 
-  MIGRATION = 'PopulateForkNetworksRange'.freeze
-  BATCH_SIZE = 100
-  DELAY_INTERVAL = 15.seconds
-
-  disable_ddl_transaction!
-
-  class ForkedProjectLink < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'forked_project_links'
-  end
-
   def up
-    say 'Populating the `fork_networks` based on existing `forked_project_links`'
-
-    queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+    say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller'
   end
 
   def down
diff --git a/db/migrate/20171207185153_add_merge_request_state_index.rb b/db/migrate/20171207185153_add_merge_request_state_index.rb
new file mode 100644
index 00000000000..72f846c5c38
--- /dev/null
+++ b/db/migrate/20171207185153_add_merge_request_state_index.rb
@@ -0,0 +1,18 @@
+class AddMergeRequestStateIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :merge_requests, [:source_project_id, :source_branch],
+                         where: "state = 'opened'",
+                         name: 'index_merge_requests_on_source_project_and_branch_state_opened'
+  end
+
+  def down
+    remove_concurrent_index_by_name :merge_requests,
+                                    'index_merge_requests_on_source_project_and_branch_state_opened'
+  end
+end
diff --git a/db/migrate/20171211145425_add_can_push_to_deploy_keys_projects.rb b/db/migrate/20171211145425_add_can_push_to_deploy_keys_projects.rb
new file mode 100644
index 00000000000..5dc723db9f9
--- /dev/null
+++ b/db/migrate/20171211145425_add_can_push_to_deploy_keys_projects.rb
@@ -0,0 +1,15 @@
+class AddCanPushToDeployKeysProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :deploy_keys_projects, :can_push, :boolean, default: false, allow_null: false
+  end
+
+  def down
+    remove_column :deploy_keys_projects, :can_push
+  end
+end
diff --git a/db/migrate/20171215113714_populate_can_push_from_deploy_keys_projects.rb b/db/migrate/20171215113714_populate_can_push_from_deploy_keys_projects.rb
new file mode 100644
index 00000000000..680855af945
--- /dev/null
+++ b/db/migrate/20171215113714_populate_can_push_from_deploy_keys_projects.rb
@@ -0,0 +1,64 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class PopulateCanPushFromDeployKeysProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+  DATABASE_NAME = Gitlab::Database.database_name
+
+  disable_ddl_transaction!
+
+  class DeploysKeyProject < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'deploy_keys_projects'
+  end
+
+  def up
+    DeploysKeyProject.each_batch(of: 10_000) do |batch|
+      start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
+
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET deploy_keys_projects.can_push = #{DATABASE_NAME}.keys.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects
+          SET can_push = keys.can_push
+          FROM keys
+          WHERE deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
+    end
+  end
+
+  def down
+    DeploysKeyProject.each_batch(of: 10_000) do |batch|
+      start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
+
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET #{DATABASE_NAME}.keys.can_push = deploy_keys_projects.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE keys
+          SET can_push = deploy_keys_projects.can_push
+          FROM deploy_keys_projects
+          WHERE deploy_keys_projects.deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
+    end
+  end
+end
diff --git a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb
index 7cf1d0cec68..d1a039ed551 100644
--- a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb
+++ b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb
@@ -9,6 +9,7 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration
     return unless Gitlab::Database.postgresql?
 
     disable_statement_timeout
+
     if Gitlab::Database.version.to_f >= 9.5
       # Allow us to hot-patch the index manually ahead of the migration
       execute "CREATE INDEX CONCURRENTLY IF NOT EXISTS #{INDEX_NAME} ON namespaces (lower(name));"
@@ -21,6 +22,7 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration
     return unless Gitlab::Database.postgresql?
 
     disable_statement_timeout
+
     if Gitlab::Database.version.to_f >= 9.2
       execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME};"
     else
diff --git a/db/migrate/20171222183504_add_jobs_cache_index_to_project.rb b/db/migrate/20171222183504_add_jobs_cache_index_to_project.rb
new file mode 100644
index 00000000000..607e9d027d7
--- /dev/null
+++ b/db/migrate/20171222183504_add_jobs_cache_index_to_project.rb
@@ -0,0 +1,13 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddJobsCacheIndexToProject < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    add_column :projects, :jobs_cache_index, :integer
+  end
+end
diff --git a/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb b/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb
new file mode 100644
index 00000000000..f942b4c062e
--- /dev/null
+++ b/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb
@@ -0,0 +1,29 @@
+class AddCommitsCountToMergeRequestDiff < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  MIGRATION = 'AddMergeRequestDiffCommitsCount'.freeze
+  BATCH_SIZE = 5000
+  DELAY_INTERVAL = 5.minutes.to_i
+
+  class MergeRequestDiff < ActiveRecord::Base
+    self.table_name = 'merge_request_diffs'
+
+    include ::EachBatch
+  end
+
+  disable_ddl_transaction!
+
+  def up
+    add_column :merge_request_diffs, :commits_count, :integer
+
+    say 'Populating the MergeRequestDiff `commits_count`'
+
+    queue_background_migration_jobs_by_range_at_intervals(MergeRequestDiff, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+  def down
+    remove_column :merge_request_diffs, :commits_count
+  end
+end
diff --git a/db/migrate/20180113220114_rework_redirect_routes_indexes.rb b/db/migrate/20180113220114_rework_redirect_routes_indexes.rb
new file mode 100644
index 00000000000..ab9971be074
--- /dev/null
+++ b/db/migrate/20180113220114_rework_redirect_routes_indexes.rb
@@ -0,0 +1,68 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class ReworkRedirectRoutesIndexes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  INDEX_NAME_UNIQUE = "index_redirect_routes_on_path_unique_text_pattern_ops"
+
+  INDEX_NAME_PERM = "index_redirect_routes_on_path_text_pattern_ops_where_permanent"
+  INDEX_NAME_TEMP = "index_redirect_routes_on_path_text_pattern_ops_where_temporary"
+
+  OLD_INDEX_NAME_PATH_TPOPS = "index_redirect_routes_on_path_text_pattern_ops"
+  OLD_INDEX_NAME_PATH_LOWER = "index_on_redirect_routes_lower_path"
+
+  def up
+    disable_statement_timeout
+
+    # this is a plain btree on a single boolean column. It'll never be
+    # selective enough to be valuable. This class is called by
+    # setup_postgresql.rake so it needs to be able to handle this
+    # index not existing.
+    if index_exists?(:redirect_routes, :permanent)
+      remove_concurrent_index(:redirect_routes, :permanent)
+    end
+
+    # If we're on MySQL then the existing index on path is ok. But on
+    # Postgres we need to clean things up:
+    return unless Gitlab::Database.postgresql?
+
+    if_not_exists = Gitlab::Database.version.to_f >= 9.5 ? "IF NOT EXISTS" : ""
+
+    # Unique index on lower(path) across both types of redirect_routes:
+    execute("CREATE UNIQUE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_UNIQUE} ON redirect_routes (lower(path) varchar_pattern_ops);")
+
+    # Make two indexes on path -- one for permanent and one for temporary routes:
+    execute("CREATE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_PERM} ON redirect_routes (lower(path) varchar_pattern_ops) where (permanent);")
+    execute("CREATE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_TEMP} ON redirect_routes (lower(path) varchar_pattern_ops) where (not permanent or permanent is null) ;")
+
+    # Remove the old indexes:
+
+    # This one needed to be on lower(path) but wasn't so it's replaced with the two above
+    execute "DROP INDEX CONCURRENTLY IF EXISTS #{OLD_INDEX_NAME_PATH_TPOPS};"
+
+    # This one isn't needed because we only ever do = and LIKE on this
+    # column so the varchar_pattern_ops index is sufficient
+    execute "DROP INDEX CONCURRENTLY IF EXISTS #{OLD_INDEX_NAME_PATH_LOWER};"
+  end
+
+  def down
+    disable_statement_timeout
+
+    add_concurrent_index(:redirect_routes, :permanent)
+
+    return unless Gitlab::Database.postgresql?
+
+    execute("CREATE INDEX CONCURRENTLY #{OLD_INDEX_NAME_PATH_TPOPS} ON redirect_routes (path varchar_pattern_ops);")
+    execute("CREATE INDEX CONCURRENTLY #{OLD_INDEX_NAME_PATH_LOWER} ON redirect_routes (LOWER(path));")
+
+    execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_UNIQUE};")
+    execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_PERM};")
+    execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_TEMP};")
+  end
+end
diff --git a/db/migrate/20180115201419_add_index_updated_at_to_issues.rb b/db/migrate/20180115201419_add_index_updated_at_to_issues.rb
new file mode 100644
index 00000000000..a5a48fc97be
--- /dev/null
+++ b/db/migrate/20180115201419_add_index_updated_at_to_issues.rb
@@ -0,0 +1,15 @@
+class AddIndexUpdatedAtToIssues < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :issues, :updated_at
+  end
+
+  def down
+    remove_concurrent_index :issues, :updated_at
+  end
+end
diff --git a/db/migrate/20180116193854_create_lfs_file_locks.rb b/db/migrate/20180116193854_create_lfs_file_locks.rb
new file mode 100644
index 00000000000..23b0c90484b
--- /dev/null
+++ b/db/migrate/20180116193854_create_lfs_file_locks.rb
@@ -0,0 +1,30 @@
+class CreateLfsFileLocks < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    create_table :lfs_file_locks do |t|
+      t.references :project, null: false, foreign_key: { on_delete: :cascade }
+      t.references :user, null: false, index: true, foreign_key: { on_delete: :cascade }
+      t.datetime :created_at, null: false
+      t.string :path, limit: 511
+    end
+
+    add_index :lfs_file_locks, [:project_id, :path], unique: true
+  end
+
+  def down
+    if foreign_keys_for(:lfs_file_locks, :project_id).any?
+      remove_foreign_key :lfs_file_locks, column: :project_id
+    end
+
+    if index_exists?(:lfs_file_locks, [:project_id, :path])
+      remove_concurrent_index :lfs_file_locks, [:project_id, :path]
+    end
+
+    drop_table :lfs_file_locks
+  end
+end
diff --git a/db/migrate/20180119135717_add_uploader_index_to_uploads.rb b/db/migrate/20180119135717_add_uploader_index_to_uploads.rb
new file mode 100644
index 00000000000..a678c3d049f
--- /dev/null
+++ b/db/migrate/20180119135717_add_uploader_index_to_uploads.rb
@@ -0,0 +1,20 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddUploaderIndexToUploads < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    remove_concurrent_index :uploads, :path
+    add_concurrent_index    :uploads, [:uploader, :path], using: :btree
+  end
+
+  def down
+    remove_concurrent_index :uploads, [:uploader, :path]
+    add_concurrent_index    :uploads, :path, using: :btree
+  end
+end
diff --git a/db/migrate/20180122162010_add_auto_devops_domain_to_application_settings.rb b/db/migrate/20180122162010_add_auto_devops_domain_to_application_settings.rb
new file mode 100644
index 00000000000..7e16cb83087
--- /dev/null
+++ b/db/migrate/20180122162010_add_auto_devops_domain_to_application_settings.rb
@@ -0,0 +1,13 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddAutoDevopsDomainToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :auto_devops_domain, :string
+  end
+end
diff --git a/db/migrate/20180125214301_create_user_callouts.rb b/db/migrate/20180125214301_create_user_callouts.rb
new file mode 100644
index 00000000000..856eff36ae0
--- /dev/null
+++ b/db/migrate/20180125214301_create_user_callouts.rb
@@ -0,0 +1,16 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreateUserCallouts < ActiveRecord::Migration
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    create_table :user_callouts do |t|
+      t.integer :feature_name, null: false
+      t.references :user, index: true, foreign_key: { on_delete: :cascade }, null: false
+    end
+
+    add_index :user_callouts, [:user_id, :feature_name], unique: true
+  end
+end
diff --git a/db/migrate/20180129193323_add_uploads_builder_context.rb b/db/migrate/20180129193323_add_uploads_builder_context.rb
new file mode 100644
index 00000000000..b3909a770ca
--- /dev/null
+++ b/db/migrate/20180129193323_add_uploads_builder_context.rb
@@ -0,0 +1,14 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddUploadsBuilderContext < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    add_column :uploads, :mount_point, :string
+    add_column :uploads, :secret, :string
+  end
+end
diff --git a/db/migrate/20180201102129_add_unique_constraint_to_trending_projects_project_id.rb b/db/migrate/20180201102129_add_unique_constraint_to_trending_projects_project_id.rb
new file mode 100644
index 00000000000..02e53b8fa8a
--- /dev/null
+++ b/db/migrate/20180201102129_add_unique_constraint_to_trending_projects_project_id.rb
@@ -0,0 +1,19 @@
+class AddUniqueConstraintToTrendingProjectsProjectId < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :trending_projects, :project_id, unique: true, name: 'index_trending_projects_on_project_id_unique'
+    remove_concurrent_index_by_name :trending_projects, 'index_trending_projects_on_project_id'
+    rename_index :trending_projects, 'index_trending_projects_on_project_id_unique', 'index_trending_projects_on_project_id'
+  end
+
+  def down
+    rename_index :trending_projects, 'index_trending_projects_on_project_id', 'index_trending_projects_on_project_id_old'
+    add_concurrent_index :trending_projects, :project_id
+    remove_concurrent_index_by_name :trending_projects, 'index_trending_projects_on_project_id_old'
+  end
+end
diff --git a/db/migrate/20180201110056_add_foreign_keys_to_todos.rb b/db/migrate/20180201110056_add_foreign_keys_to_todos.rb
new file mode 100644
index 00000000000..b7c40f8c01a
--- /dev/null
+++ b/db/migrate/20180201110056_add_foreign_keys_to_todos.rb
@@ -0,0 +1,38 @@
+class AddForeignKeysToTodos < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  class Todo < ActiveRecord::Base
+    self.table_name = 'todos'
+    include EachBatch
+  end
+
+  BATCH_SIZE = 1000
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    Todo.where('NOT EXISTS ( SELECT true FROM users WHERE id=todos.user_id )').each_batch(of: BATCH_SIZE) do |batch|
+      batch.delete_all
+    end
+
+    Todo.where('NOT EXISTS ( SELECT true FROM users WHERE id=todos.author_id )').each_batch(of: BATCH_SIZE) do |batch|
+      batch.delete_all
+    end
+
+    Todo.where('note_id IS NOT NULL AND NOT EXISTS ( SELECT true FROM notes WHERE id=todos.note_id )').each_batch(of: BATCH_SIZE) do |batch|
+      batch.delete_all
+    end
+
+    add_concurrent_foreign_key :todos, :users, column: :user_id, on_delete: :cascade
+    add_concurrent_foreign_key :todos, :users, column: :author_id, on_delete: :cascade
+    add_concurrent_foreign_key :todos, :notes, column: :note_id, on_delete: :cascade
+  end
+
+  def down
+    remove_foreign_key :todos, :users
+    remove_foreign_key :todos, column: :author_id
+    remove_foreign_key :todos, :notes
+  end
+end
diff --git a/db/migrate/20180201145907_migrate_remaining_issues_closed_at.rb b/db/migrate/20180201145907_migrate_remaining_issues_closed_at.rb
new file mode 100644
index 00000000000..5a36dec6a9a
--- /dev/null
+++ b/db/migrate/20180201145907_migrate_remaining_issues_closed_at.rb
@@ -0,0 +1,55 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class MigrateRemainingIssuesClosedAt < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class Issue < ActiveRecord::Base
+    self.table_name = 'issues'
+    include EachBatch
+  end
+
+  def up
+    Gitlab::BackgroundMigration.steal('CopyColumn')
+    Gitlab::BackgroundMigration.steal('CleanupConcurrentTypeChange')
+
+    if migrate_column_type?
+      if closed_at_for_type_change_exists?
+        migrate_remaining_rows
+      else
+        # Due to some EE merge problems some environments may not have the
+        # "closed_at_for_type_change" column. If this is the case we have no
+        # other option than to migrate the data _right now_.
+        change_column_type_concurrently(:issues, :closed_at, :datetime_with_timezone)
+        cleanup_concurrent_column_type_change(:issues, :closed_at)
+      end
+    end
+  end
+
+  def down
+    # Previous migrations already revert the changes made here.
+  end
+
+  def migrate_remaining_rows
+    Issue.where('closed_at_for_type_change IS NULL AND closed_at IS NOT NULL').each_batch do |batch|
+      batch.update_all('closed_at_for_type_change = closed_at')
+    end
+
+    cleanup_concurrent_column_type_change(:issues, :closed_at)
+  end
+
+  def migrate_column_type?
+    # Some environments may have already executed the previous version of this
+    # migration, thus we don't need to migrate those environments again.
+    column_for('issues', 'closed_at').type == :datetime # rubocop:disable Migration/Datetime
+  end
+
+  def closed_at_for_type_change_exists?
+    columns('issues').any? { |col| col.name == 'closed_at_for_type_change' }
+  end
+end
diff --git a/db/migrate/20180206200543_reset_events_primary_key_sequence.rb b/db/migrate/20180206200543_reset_events_primary_key_sequence.rb
new file mode 100644
index 00000000000..eb5c4a6a1e7
--- /dev/null
+++ b/db/migrate/20180206200543_reset_events_primary_key_sequence.rb
@@ -0,0 +1,35 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class ResetEventsPrimaryKeySequence < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  class Event < ActiveRecord::Base
+    self.table_name = 'events'
+  end
+
+  def up
+    if Gitlab::Database.postgresql?
+      reset_primary_key_for_postgresql
+    else
+      reset_primary_key_for_mysql
+    end
+  end
+
+  def down
+    # No-op
+  end
+
+  def reset_primary_key_for_postgresql
+    reset_pk_sequence!(Event.table_name)
+  end
+
+  def reset_primary_key_for_mysql
+    amount = Event.pluck('COALESCE(MAX(id), 1)').first
+
+    execute "ALTER TABLE #{Event.table_name} AUTO_INCREMENT = #{amount}"
+  end
+end
diff --git a/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb b/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb
index da0fcda87a6..17ad7de065d 100644
--- a/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb
+++ b/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb
@@ -31,6 +31,7 @@ class RenameUsersWithRenamedNamespace < ActiveRecord::Migration
       predicate = namespaces[:owner_id].eq(users[:id])
                     .and(namespaces[:type].eq(nil))
                     .and(users[:username].matches(path))
+
       update_sql = if Gitlab::Database.postgresql?
                      "UPDATE users SET username = namespaces.path "\
                      "FROM namespaces WHERE #{predicate.to_sql}"
diff --git a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb
new file mode 100644
index 00000000000..11b581e4b57
--- /dev/null
+++ b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb
@@ -0,0 +1,151 @@
+class MigrateKubernetesServiceToNewClustersArchitectures < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME = 'KubernetesService'.freeze
+
+  disable_ddl_transaction!
+
+  class Project < ActiveRecord::Base
+    self.table_name = 'projects'
+
+    has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject'
+    has_many :clusters, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+    has_many :services, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service'
+    has_one :kubernetes_service, -> { where(category: 'deployment', type: 'KubernetesService') }, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service', inverse_of: :project, foreign_key: :project_id
+  end
+
+  class Cluster < ActiveRecord::Base
+    self.table_name = 'clusters'
+
+    has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject'
+    has_many :projects, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project'
+    has_one :platform_kubernetes, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::PlatformsKubernetes'
+
+    accepts_nested_attributes_for :platform_kubernetes
+
+    enum platform_type: {
+      kubernetes: 1
+    }
+
+    enum provider_type: {
+      user: 0,
+      gcp: 1
+    }
+  end
+
+  class ClustersProject < ActiveRecord::Base
+    self.table_name = 'cluster_projects'
+
+    belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+    belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project'
+  end
+
+  class PlatformsKubernetes < ActiveRecord::Base
+    self.table_name = 'cluster_platforms_kubernetes'
+
+    belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+
+    attr_encrypted :token,
+      mode: :per_attribute_iv,
+      key: Gitlab::Application.secrets.db_key_base,
+      algorithm: 'aes-256-cbc'
+  end
+
+  class Service < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'services'
+    self.inheritance_column = :_type_disabled # Disable STI, otherwise KubernetesModel will be looked up
+
+    belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project', foreign_key: :project_id
+
+    scope :unmanaged_kubernetes_service, -> do
+      joins('LEFT JOIN projects ON projects.id = services.project_id')
+      .joins('LEFT JOIN cluster_projects ON cluster_projects.project_id = projects.id')
+      .joins('LEFT JOIN cluster_platforms_kubernetes ON cluster_platforms_kubernetes.cluster_id = cluster_projects.cluster_id')
+      .where(category: 'deployment', type: 'KubernetesService', template: false)
+      .where("services.properties LIKE '%api_url%'")
+      .where("(services.properties NOT LIKE CONCAT('%', cluster_platforms_kubernetes.api_url, '%')) OR cluster_platforms_kubernetes.api_url IS NULL")
+      .group(:id)
+      .order(id: :asc)
+    end
+
+    scope :kubernetes_service_without_template, -> do
+      where(category: 'deployment', type: 'KubernetesService', template: false)
+    end
+
+    def api_url
+      parsed_properties['api_url']
+    end
+
+    def ca_pem
+      parsed_properties['ca_pem']
+    end
+
+    def namespace
+      parsed_properties['namespace']
+    end
+
+    def token
+      parsed_properties['token']
+    end
+
+    private
+
+    def parsed_properties
+      @parsed_properties ||= JSON.parse(self.properties)
+    end
+  end
+
+  def find_dedicated_environement_scope(project)
+    environment_scopes = project.clusters.map(&:environment_scope)
+
+    return '*' if environment_scopes.exclude?('*') # KubernetesService should be added as a default cluster (environment_scope: '*') at first place
+    return 'migrated/*' if environment_scopes.exclude?('migrated/*') # If it's conflicted, the KubernetesService added as a migrated cluster
+
+    unique_iid = 0
+
+    # If it's still conflicted, finding an unique environment scope incrementaly
+    loop do
+      candidate = "migrated#{unique_iid}/*"
+      return candidate if environment_scopes.exclude?(candidate)
+
+      unique_iid += 1
+    end
+  end
+
+  def up
+    ActiveRecord::Base.transaction do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service
+        .unmanaged_kubernetes_service.find_each(batch_size: 1) do |kubernetes_service|
+        MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create(
+          enabled: kubernetes_service.active,
+          user_id: nil, # KubernetesService doesn't have
+          name: DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME,
+          provider_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.provider_types[:user],
+          platform_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.platform_types[:kubernetes],
+          projects: [kubernetes_service.project],
+          environment_scope: find_dedicated_environement_scope(kubernetes_service.project),
+          platform_kubernetes_attributes: {
+            api_url: kubernetes_service.api_url,
+            ca_cert: kubernetes_service.ca_pem,
+            namespace: kubernetes_service.namespace,
+            username: nil, # KubernetesService doesn't have
+            encrypted_password: nil, # KubernetesService doesn't have
+            encrypted_password_iv: nil, # KubernetesService doesn't have
+            token: kubernetes_service.token # encrypted_token and encrypted_token_iv
+          } )
+      end
+    end
+
+    MigrateKubernetesServiceToNewClustersArchitectures::Service
+      .kubernetes_service_without_template.each_batch(of: 100) do |kubernetes_service|
+      kubernetes_service.update_all(active: false)
+    end
+  end
+
+  def down
+    # noop
+  end
+end
diff --git a/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb b/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb
index 05430efe1f6..26f917d5a1e 100644
--- a/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb
+++ b/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb
@@ -3,22 +3,8 @@ class RescheduleForkNetworkCreation < ActiveRecord::Migration
 
   DOWNTIME = false
 
-  MIGRATION = 'PopulateForkNetworksRange'.freeze
-  BATCH_SIZE = 100
-  DELAY_INTERVAL = 15.seconds
-
-  disable_ddl_transaction!
-
-  class ForkedProjectLink < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'forked_project_links'
-  end
-
   def up
-    say 'Populating the `fork_networks` based on existing `forked_project_links`'
-
-    queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+    say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller'
   end
 
   def down
diff --git a/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb
index 547cc68e10e..fce1829c982 100644
--- a/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb
+++ b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb
@@ -15,8 +15,6 @@ class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migratio
   end
 
   def up
-    merge_requests = MergeRequest.where("id IN (#{updatable_merge_requests_union_sql})").reorder(:id)
-
     say 'Scheduling `PopulateMergeRequestMetricsWithEventsData` jobs'
     # It will update around 4_000_000 records in batches of 10_000 merge
     # requests (running between 10 minutes) and should take around 66 hours to complete.
@@ -25,7 +23,7 @@ class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migratio
     #
     # More information about the updates in `PopulateMergeRequestMetricsWithEventsData` class.
     #
-    merge_requests.each_batch(of: BATCH_SIZE) do |relation, index|
+    MergeRequest.all.each_batch(of: BATCH_SIZE) do |relation, index|
       range = relation.pluck('MIN(id)', 'MAX(id)').first
 
       BackgroundMigrationWorker.perform_in(index * 10.minutes, MIGRATION, range)
@@ -37,32 +35,4 @@ class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migratio
     execute "update merge_request_metrics set latest_closed_by_id = null"
     execute "update merge_request_metrics set merged_by_id = null"
   end
-
-  private
-
-  # On staging:
-  # Planning time: 0.682 ms
-  # Execution time: 22033.158 ms
-  #
-  def updatable_merge_requests_union_sql
-    metrics_not_exists_clause =
-      'NOT EXISTS (SELECT 1 FROM merge_request_metrics WHERE merge_request_metrics.merge_request_id = merge_requests.id)'
-
-    without_metrics_data = <<-SQL.strip_heredoc
-      merge_request_metrics.merged_by_id IS NULL OR
-      merge_request_metrics.latest_closed_by_id IS NULL OR
-      merge_request_metrics.latest_closed_at IS NULL
-    SQL
-
-    mrs_without_metrics_record = MergeRequest
-      .where(metrics_not_exists_clause)
-      .select(:id)
-
-    mrs_without_events_data = MergeRequest
-      .joins('INNER JOIN merge_request_metrics ON merge_requests.id = merge_request_metrics.merge_request_id')
-      .where(without_metrics_data)
-      .select(:id)
-
-    Gitlab::SQL::Union.new([mrs_without_metrics_record, mrs_without_events_data]).to_sql
-  end
 end
diff --git a/db/post_migrate/20171207150300_remove_project_labels_group_id_copy.rb b/db/post_migrate/20171207150300_remove_project_labels_group_id_copy.rb
new file mode 100644
index 00000000000..2f339172eeb
--- /dev/null
+++ b/db/post_migrate/20171207150300_remove_project_labels_group_id_copy.rb
@@ -0,0 +1,21 @@
+# Copy of 20180202111106 - this one should run before 20171207150343 to fix issues related to
+# the removal of groups with labels.
+
+class RemoveProjectLabelsGroupIdCopy < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    # rubocop:disable Migration/UpdateColumnInBatches
+    update_column_in_batches(:labels, :group_id, nil) do |table, query|
+      query.where(table[:type].eq('ProjectLabel').and(table[:group_id].not_eq(nil)))
+    end
+    # rubocop:enable Migration/UpdateColumnInBatches
+  end
+
+  def down
+  end
+end
diff --git a/db/post_migrate/20171207150343_remove_soft_removed_objects.rb b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb
new file mode 100644
index 00000000000..3e2dedfdd6a
--- /dev/null
+++ b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb
@@ -0,0 +1,208 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveSoftRemovedObjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  module SoftRemoved
+    extend ActiveSupport::Concern
+
+    included do
+      scope :soft_removed, -> { where('deleted_at IS NOT NULL') }
+    end
+  end
+
+  class User < ActiveRecord::Base
+    self.table_name = 'users'
+
+    include EachBatch
+  end
+
+  class Issue < ActiveRecord::Base
+    self.table_name = 'issues'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class MergeRequest < ActiveRecord::Base
+    self.table_name = 'merge_requests'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class Namespace < ActiveRecord::Base
+    self.table_name = 'namespaces'
+
+    include EachBatch
+    include SoftRemoved
+
+    scope :soft_removed_personal, -> { soft_removed.where(type: nil) }
+    scope :soft_removed_group, -> { soft_removed.where(type: 'Group') }
+  end
+
+  class Route < ActiveRecord::Base
+    self.table_name = 'routes'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class Project < ActiveRecord::Base
+    self.table_name = 'projects'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class CiPipelineSchedule < ActiveRecord::Base
+    self.table_name = 'ci_pipeline_schedules'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class CiTrigger < ActiveRecord::Base
+    self.table_name = 'ci_triggers'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  MODELS = [Issue, MergeRequest, CiPipelineSchedule, CiTrigger].freeze
+
+  def up
+    disable_statement_timeout
+
+    remove_personal_routes
+    remove_personal_namespaces
+    remove_group_namespaces
+    remove_simple_soft_removed_rows
+  end
+
+  def down
+    # The data removed by this migration can't be restored in an automated way.
+  end
+
+  def remove_simple_soft_removed_rows
+    create_temporary_indexes
+
+    MODELS.each do |model|
+      say_with_time("Removing soft removed rows from #{model.table_name}") do
+        model.soft_removed.each_batch do |batch, index|
+          batch.delete_all
+        end
+      end
+    end
+  ensure
+    remove_temporary_indexes
+  end
+
+  def create_temporary_indexes
+    MODELS.each do |model|
+      index_name = temporary_index_name_for(model)
+
+      # Without this index the removal process can take a very long time. For
+      # example, getting the next ID of a batch for the `issues` table in
+      # staging would take between 15 and 20 seconds.
+      next if temporary_index_exists?(model)
+
+      say_with_time("Creating temporary index #{index_name}") do
+        add_concurrent_index(
+          model.table_name,
+          [:deleted_at, :id],
+          name: index_name,
+          where: 'deleted_at IS NOT NULL'
+        )
+      end
+    end
+  end
+
+  def remove_temporary_indexes
+    MODELS.each do |model|
+      index_name = temporary_index_name_for(model)
+
+      next unless temporary_index_exists?(model)
+
+      say_with_time("Removing temporary index #{index_name}") do
+        remove_concurrent_index_by_name(model.table_name, index_name)
+      end
+    end
+  end
+
+  def temporary_index_name_for(model)
+    "index_on_#{model.table_name}_tmp"
+  end
+
+  def temporary_index_exists?(model)
+    index_name = temporary_index_name_for(model)
+
+    index_exists?(model.table_name, [:deleted_at, :id], name: index_name)
+  end
+
+  def remove_personal_namespaces
+    # Some personal namespaces are left behind in case of GitLab.com. In these
+    # cases the associated data such as the projects and users has already been
+    # removed.
+    Namespace.soft_removed_personal.each_batch do |batch|
+      batch.delete_all
+    end
+  end
+
+  def remove_group_namespaces
+    admin_id = id_for_admin_user
+
+    unless admin_id
+      say 'Not scheduling soft removed groups for removal as no admin user ' \
+        'could be found. You will need to remove any such groups manually.'
+
+      return
+    end
+
+    # Left over groups can't be easily removed because we may also need to
+    # remove memberships, repositories, and other associated data. As a result
+    # we'll just schedule a Sidekiq job to remove these.
+    #
+    # As of January 5th, 2018 there are 36 groups that will be removed using
+    # this code.
+    Namespace.select(:id).soft_removed_group.each_batch(of: 10) do |batch, index|
+      batch.each do |ns|
+        schedule_group_removal(index * 5.minutes, ns.id, admin_id)
+      end
+    end
+  end
+
+  def schedule_group_removal(delay, group_id, user_id)
+    if migrate_inline?
+      GroupDestroyWorker.new.perform(group_id, user_id)
+    else
+      GroupDestroyWorker.perform_in(delay, group_id, user_id)
+    end
+  end
+
+  def remove_personal_routes
+    namespaces = Namespace.select(1)
+      .soft_removed
+      .where('namespaces.type IS NULL')
+      .where('routes.source_type = ?', 'Namespace')
+      .where('routes.source_id = namespaces.id')
+
+    Route.where('EXISTS (?)', namespaces).each_batch do |batch|
+      batch.delete_all
+    end
+  end
+
+  def id_for_admin_user
+    User.where(admin: true).limit(1).pluck(:id).first
+  end
+
+  def migrate_inline?
+    Rails.env.test? || Rails.env.development?
+  end
+end
diff --git a/db/post_migrate/20171207150344_remove_deleted_at_columns.rb b/db/post_migrate/20171207150344_remove_deleted_at_columns.rb
new file mode 100644
index 00000000000..154d7a1b926
--- /dev/null
+++ b/db/post_migrate/20171207150344_remove_deleted_at_columns.rb
@@ -0,0 +1,31 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveDeletedAtColumns < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  TABLES = %i[issues merge_requests namespaces ci_pipeline_schedules ci_triggers].freeze
+  COLUMN = :deleted_at
+
+  def up
+    TABLES.each do |table|
+      remove_column(table, COLUMN) if column_exists?(table, COLUMN)
+    end
+  end
+
+  def down
+    TABLES.each do |table|
+      unless column_exists?(table, COLUMN)
+        add_column(table, COLUMN, :datetime_with_timezone)
+      end
+
+      unless index_exists?(table, COLUMN)
+        add_concurrent_index(table, COLUMN)
+      end
+    end
+  end
+end
diff --git a/db/post_migrate/20171215121205_post_populate_can_push_from_deploy_keys_projects.rb b/db/post_migrate/20171215121205_post_populate_can_push_from_deploy_keys_projects.rb
new file mode 100644
index 00000000000..3a5850df3db
--- /dev/null
+++ b/db/post_migrate/20171215121205_post_populate_can_push_from_deploy_keys_projects.rb
@@ -0,0 +1,63 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class PostPopulateCanPushFromDeployKeysProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DATABASE_NAME = Gitlab::Database.database_name
+
+  disable_ddl_transaction!
+
+  class DeploysKeyProject < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'deploy_keys_projects'
+  end
+
+  def up
+    DeploysKeyProject.each_batch(of: 10_000) do |batch|
+      start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
+
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET deploy_keys_projects.can_push = #{DATABASE_NAME}.keys.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects
+          SET can_push = keys.can_push
+          FROM keys
+          WHERE deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
+    end
+  end
+
+  def down
+    DeploysKeyProject.each_batch(of: 10_000) do |batch|
+      start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
+
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET #{DATABASE_NAME}.keys.can_push = deploy_keys_projects.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE keys
+          SET can_push = deploy_keys_projects.can_push
+          FROM deploy_keys_projects
+          WHERE deploy_keys_projects.deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
+    end
+  end
+end
diff --git a/db/post_migrate/20171215121259_remove_can_push_from_keys.rb b/db/post_migrate/20171215121259_remove_can_push_from_keys.rb
new file mode 100644
index 00000000000..0599811d986
--- /dev/null
+++ b/db/post_migrate/20171215121259_remove_can_push_from_keys.rb
@@ -0,0 +1,17 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveCanPushFromKeys < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  disable_ddl_transaction!
+
+  def up
+    remove_column :keys, :can_push
+  end
+
+  def down
+    add_column_with_default :keys, :can_push, :boolean, default: false, allow_null: false
+  end
+end
diff --git a/db/post_migrate/20180119121225_remove_redundant_pipeline_stages.rb b/db/post_migrate/20180119121225_remove_redundant_pipeline_stages.rb
new file mode 100644
index 00000000000..61ea85eb2a7
--- /dev/null
+++ b/db/post_migrate/20180119121225_remove_redundant_pipeline_stages.rb
@@ -0,0 +1,66 @@
+class RemoveRedundantPipelineStages < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up(attempts: 100)
+    remove_redundant_pipeline_stages!
+    remove_outdated_index!
+    add_unique_index!
+  rescue ActiveRecord::RecordNotUnique
+    retry if (attempts -= 1) > 0
+
+    raise StandardError, <<~EOS
+      Failed to add an unique index to ci_stages, despite retrying the
+      migration 100 times.
+
+      See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/16580.
+    EOS
+  end
+
+  def down
+    remove_concurrent_index :ci_stages, [:pipeline_id, :name], unique: true
+    add_concurrent_index :ci_stages, [:pipeline_id, :name]
+  end
+
+  private
+
+  def remove_outdated_index!
+    return unless index_exists?(:ci_stages, [:pipeline_id, :name])
+
+    remove_concurrent_index :ci_stages, [:pipeline_id, :name]
+  end
+
+  def add_unique_index!
+    add_concurrent_index :ci_stages, [:pipeline_id, :name], unique: true
+  end
+
+  def remove_redundant_pipeline_stages!
+    disable_statement_timeout
+
+    redundant_stages_ids = <<~SQL
+      SELECT id FROM ci_stages WHERE (pipeline_id, name) IN (
+        SELECT pipeline_id, name FROM ci_stages
+          GROUP BY pipeline_id, name HAVING COUNT(*) > 1
+      )
+    SQL
+
+    execute <<~SQL
+      UPDATE ci_builds SET stage_id = NULL WHERE stage_id IN (#{redundant_stages_ids})
+    SQL
+
+    if Gitlab::Database.postgresql?
+      execute <<~SQL
+        DELETE FROM ci_stages WHERE id IN (#{redundant_stages_ids})
+      SQL
+    else # We can't modify a table we are selecting from on MySQL
+      execute <<~SQL
+        DELETE a FROM ci_stages AS a, ci_stages AS b
+          WHERE a.pipeline_id = b.pipeline_id AND a.name = b.name
+            AND a.id <> b.id
+      SQL
+    end
+  end
+end
diff --git a/db/post_migrate/20180202111106_remove_project_labels_group_id.rb b/db/post_migrate/20180202111106_remove_project_labels_group_id.rb
new file mode 100644
index 00000000000..db7fd0d167d
--- /dev/null
+++ b/db/post_migrate/20180202111106_remove_project_labels_group_id.rb
@@ -0,0 +1,19 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveProjectLabelsGroupId < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    update_column_in_batches(:labels, :group_id, nil) do |table, query|
+      query.where(table[:type].eq('ProjectLabel').and(table[:group_id].not_eq(nil)))
+    end
+  end
+
+  def down
+  end
+end
diff --git a/db/post_migrate/20180204200836_change_author_id_to_not_null_in_todos.rb b/db/post_migrate/20180204200836_change_author_id_to_not_null_in_todos.rb
new file mode 100644
index 00000000000..92c32feebf7
--- /dev/null
+++ b/db/post_migrate/20180204200836_change_author_id_to_not_null_in_todos.rb
@@ -0,0 +1,26 @@
+class ChangeAuthorIdToNotNullInTodos < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  class Todo < ActiveRecord::Base
+    self.table_name = 'todos'
+    include EachBatch
+  end
+
+  BATCH_SIZE = 1000
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    Todo.where(author_id: nil).each_batch(of: BATCH_SIZE) do |batch|
+      batch.delete_all
+    end
+
+    change_column_null :todos, :author_id, false
+  end
+
+  def down
+    change_column_null :todos, :author_id, true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1eba30b28c8..d07a4c31618 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180105101928) do
+ActiveRecord::Schema.define(version: 20180206200543) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -154,6 +154,8 @@ ActiveRecord::Schema.define(version: 20180105101928) do
     t.integer "gitaly_timeout_default", default: 55, null: false
     t.integer "gitaly_timeout_medium", default: 30, null: false
     t.integer "gitaly_timeout_fast", default: 10, null: false
+    t.boolean "authorized_keys_enabled", default: true, null: false
+    t.string "auto_devops_domain"
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -356,7 +358,6 @@ ActiveRecord::Schema.define(version: 20180105101928) do
     t.integer "project_id"
     t.integer "owner_id"
     t.boolean "active", default: true
-    t.datetime "deleted_at"
     t.datetime "created_at"
     t.datetime "updated_at"
   end
@@ -450,7 +451,7 @@ ActiveRecord::Schema.define(version: 20180105101928) do
     t.integer "lock_version"
   end
 
-  add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", using: :btree
+  add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", unique: true, using: :btree
   add_index "ci_stages", ["pipeline_id"], name: "index_ci_stages_on_pipeline_id", using: :btree
   add_index "ci_stages", ["project_id"], name: "index_ci_stages_on_project_id", using: :btree
 
@@ -466,7 +467,6 @@ ActiveRecord::Schema.define(version: 20180105101928) do
 
   create_table "ci_triggers", force: :cascade do |t|
     t.string "token"
-    t.datetime "deleted_at"
     t.datetime "created_at"
     t.datetime "updated_at"
     t.integer "project_id"
@@ -627,6 +627,7 @@ ActiveRecord::Schema.define(version: 20180105101928) do
     t.integer "project_id", null: false
     t.datetime "created_at"
     t.datetime "updated_at"
+    t.boolean "can_push", default: false, null: false
   end
 
   add_index "deploy_keys_projects", ["project_id"], name: "index_deploy_keys_projects_on_project_id", using: :btree
@@ -860,7 +861,6 @@ ActiveRecord::Schema.define(version: 20180105101928) do
     t.integer "iid"
     t.integer "updated_by_id"
     t.boolean "confidential", default: false, null: false
-    t.datetime "deleted_at"
     t.date "due_date"
     t.integer "moved_to_id"
     t.integer "lock_version"
@@ -877,7 +877,6 @@ ActiveRecord::Schema.define(version: 20180105101928) do
 
   add_index "issues", ["author_id"], name: "index_issues_on_author_id", using: :btree
   add_index "issues", ["confidential"], name: "index_issues_on_confidential", using: :btree
-  add_index "issues", ["deleted_at"], name: "index_issues_on_deleted_at", using: :btree
   add_index "issues", ["description"], name: "index_issues_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
   add_index "issues", ["milestone_id"], name: "index_issues_on_milestone_id", using: :btree
   add_index "issues", ["moved_to_id"], name: "index_issues_on_moved_to_id", where: "(moved_to_id IS NOT NULL)", using: :btree
@@ -888,6 +887,7 @@ ActiveRecord::Schema.define(version: 20180105101928) do
   add_index "issues", ["relative_position"], name: "index_issues_on_relative_position", using: :btree
   add_index "issues", ["state"], name: "index_issues_on_state", using: :btree
   add_index "issues", ["title"], name: "index_issues_on_title_trigram", using: :gin, opclasses: {"title"=>"gin_trgm_ops"}
+  add_index "issues", ["updated_at"], name: "index_issues_on_updated_at", using: :btree
   add_index "issues", ["updated_by_id"], name: "index_issues_on_updated_by_id", where: "(updated_by_id IS NOT NULL)", using: :btree
 
   create_table "keys", force: :cascade do |t|
@@ -899,7 +899,6 @@ ActiveRecord::Schema.define(version: 20180105101928) do
     t.string "type"
     t.string "fingerprint"
     t.boolean "public", default: false, null: false
-    t.boolean "can_push", default: false, null: false
     t.datetime "last_used_at"
   end
 
@@ -948,6 +947,16 @@ ActiveRecord::Schema.define(version: 20180105101928) do
   add_index "labels", ["title"], name: "index_labels_on_title", using: :btree
   add_index "labels", ["type", "project_id"], name: "index_labels_on_type_and_project_id", using: :btree
 
+  create_table "lfs_file_locks", force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.integer "user_id", null: false
+    t.datetime "created_at", null: false
+    t.string "path", limit: 511
+  end
+
+  add_index "lfs_file_locks", ["project_id", "path"], name: "index_lfs_file_locks_on_project_id_and_path", unique: true, using: :btree
+  add_index "lfs_file_locks", ["user_id"], name: "index_lfs_file_locks_on_user_id", using: :btree
+
   create_table "lfs_objects", force: :cascade do |t|
     t.string "oid", null: false
     t.integer "size", limit: 8, null: false
@@ -1016,6 +1025,7 @@ ActiveRecord::Schema.define(version: 20180105101928) do
   end
 
   add_index "merge_request_diff_commits", ["merge_request_diff_id", "relative_order"], name: "index_merge_request_diff_commits_on_mr_diff_id_and_order", unique: true, using: :btree
+  add_index "merge_request_diff_commits", ["sha"], name: "index_merge_request_diff_commits_on_sha", using: :btree
 
   create_table "merge_request_diff_files", id: false, force: :cascade do |t|
     t.integer "merge_request_diff_id", null: false
@@ -1043,6 +1053,7 @@ ActiveRecord::Schema.define(version: 20180105101928) do
     t.string "real_size"
     t.string "head_commit_sha"
     t.string "start_commit_sha"
+    t.integer "commits_count"
   end
 
   add_index "merge_request_diffs", ["merge_request_id", "id"], name: "index_merge_request_diffs_on_merge_request_id_and_id", using: :btree
@@ -1086,7 +1097,6 @@ ActiveRecord::Schema.define(version: 20180105101928) do
     t.boolean "merge_when_pipeline_succeeds", default: false, null: false
     t.integer "merge_user_id"
     t.string "merge_commit_sha"
-    t.datetime "deleted_at"
     t.string "in_progress_merge_commit_sha"
     t.integer "lock_version"
     t.text "title_html"
@@ -1105,13 +1115,13 @@ ActiveRecord::Schema.define(version: 20180105101928) do
   add_index "merge_requests", ["assignee_id"], name: "index_merge_requests_on_assignee_id", using: :btree
   add_index "merge_requests", ["author_id"], name: "index_merge_requests_on_author_id", using: :btree
   add_index "merge_requests", ["created_at"], name: "index_merge_requests_on_created_at", using: :btree
-  add_index "merge_requests", ["deleted_at"], name: "index_merge_requests_on_deleted_at", using: :btree
   add_index "merge_requests", ["description"], name: "index_merge_requests_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
   add_index "merge_requests", ["head_pipeline_id"], name: "index_merge_requests_on_head_pipeline_id", using: :btree
   add_index "merge_requests", ["latest_merge_request_diff_id"], name: "index_merge_requests_on_latest_merge_request_diff_id", using: :btree
   add_index "merge_requests", ["merge_user_id"], name: "index_merge_requests_on_merge_user_id", where: "(merge_user_id IS NOT NULL)", using: :btree
   add_index "merge_requests", ["milestone_id"], name: "index_merge_requests_on_milestone_id", using: :btree
   add_index "merge_requests", ["source_branch"], name: "index_merge_requests_on_source_branch", using: :btree
+  add_index "merge_requests", ["source_project_id", "source_branch"], name: "index_merge_requests_on_source_project_and_branch_state_opened", where: "((state)::text = 'opened'::text)", using: :btree
   add_index "merge_requests", ["source_project_id", "source_branch"], name: "index_merge_requests_on_source_project_id_and_source_branch", using: :btree
   add_index "merge_requests", ["target_branch"], name: "index_merge_requests_on_target_branch", using: :btree
   add_index "merge_requests", ["target_project_id", "iid"], name: "index_merge_requests_on_target_project_id_and_iid", unique: true, using: :btree
@@ -1165,7 +1175,6 @@ ActiveRecord::Schema.define(version: 20180105101928) do
     t.boolean "share_with_group_lock", default: false
     t.integer "visibility_level", default: 20, null: false
     t.boolean "request_access_enabled", default: false, null: false
-    t.datetime "deleted_at"
     t.text "description_html"
     t.boolean "lfs_enabled"
     t.integer "parent_id"
@@ -1175,7 +1184,6 @@ ActiveRecord::Schema.define(version: 20180105101928) do
   end
 
   add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree
-  add_index "namespaces", ["deleted_at"], name: "index_namespaces_on_deleted_at", using: :btree
   add_index "namespaces", ["name", "parent_id"], name: "index_namespaces_on_name_and_parent_id", unique: true, using: :btree
   add_index "namespaces", ["name"], name: "index_namespaces_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"}
   add_index "namespaces", ["owner_id"], name: "index_namespaces_on_owner_id", using: :btree
@@ -1451,6 +1459,7 @@ ActiveRecord::Schema.define(version: 20180105101928) do
     t.boolean "repository_read_only"
     t.boolean "merge_requests_ff_only_enabled", default: false
     t.boolean "merge_requests_rebase_enabled", default: false, null: false
+    t.integer "jobs_cache_index"
   end
 
   add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
@@ -1541,8 +1550,6 @@ ActiveRecord::Schema.define(version: 20180105101928) do
   end
 
   add_index "redirect_routes", ["path"], name: "index_redirect_routes_on_path", unique: true, using: :btree
-  add_index "redirect_routes", ["path"], name: "index_redirect_routes_on_path_text_pattern_ops", using: :btree, opclasses: {"path"=>"varchar_pattern_ops"}
-  add_index "redirect_routes", ["permanent"], name: "index_redirect_routes_on_permanent", using: :btree
   add_index "redirect_routes", ["source_type", "source_id"], name: "index_redirect_routes_on_source_type_and_source_id", using: :btree
 
   create_table "releases", force: :cascade do |t|
@@ -1711,7 +1718,7 @@ ActiveRecord::Schema.define(version: 20180105101928) do
     t.integer "project_id", null: false
     t.integer "target_id"
     t.string "target_type", null: false
-    t.integer "author_id"
+    t.integer "author_id", null: false
     t.integer "action", null: false
     t.string "state", null: false
     t.datetime "created_at"
@@ -1731,7 +1738,7 @@ ActiveRecord::Schema.define(version: 20180105101928) do
     t.integer "project_id", null: false
   end
 
-  add_index "trending_projects", ["project_id"], name: "index_trending_projects_on_project_id", using: :btree
+  add_index "trending_projects", ["project_id"], name: "index_trending_projects_on_project_id", unique: true, using: :btree
 
   create_table "u2f_registrations", force: :cascade do |t|
     t.text "certificate"
@@ -1755,11 +1762,13 @@ ActiveRecord::Schema.define(version: 20180105101928) do
     t.string "model_type"
     t.string "uploader", null: false
     t.datetime "created_at", null: false
+    t.string "mount_point"
+    t.string "secret"
   end
 
   add_index "uploads", ["checksum"], name: "index_uploads_on_checksum", using: :btree
   add_index "uploads", ["model_id", "model_type"], name: "index_uploads_on_model_id_and_model_type", using: :btree
-  add_index "uploads", ["path"], name: "index_uploads_on_path", using: :btree
+  add_index "uploads", ["uploader", "path"], name: "index_uploads_on_uploader_and_path", using: :btree
 
   create_table "user_agent_details", force: :cascade do |t|
     t.string "user_agent", null: false
@@ -1773,6 +1782,14 @@ ActiveRecord::Schema.define(version: 20180105101928) do
 
   add_index "user_agent_details", ["subject_id", "subject_type"], name: "index_user_agent_details_on_subject_id_and_subject_type", using: :btree
 
+  create_table "user_callouts", force: :cascade do |t|
+    t.integer "feature_name", null: false
+    t.integer "user_id", null: false
+  end
+
+  add_index "user_callouts", ["user_id", "feature_name"], name: "index_user_callouts_on_user_id_and_feature_name", unique: true, using: :btree
+  add_index "user_callouts", ["user_id"], name: "index_user_callouts_on_user_id", using: :btree
+
   create_table "user_custom_attributes", force: :cascade do |t|
     t.datetime_with_timezone "created_at", null: false
     t.datetime_with_timezone "updated_at", null: false
@@ -1991,6 +2008,8 @@ ActiveRecord::Schema.define(version: 20180105101928) do
   add_foreign_key "label_priorities", "projects", on_delete: :cascade
   add_foreign_key "labels", "namespaces", column: "group_id", on_delete: :cascade
   add_foreign_key "labels", "projects", name: "fk_7de4989a69", on_delete: :cascade
+  add_foreign_key "lfs_file_locks", "projects", on_delete: :cascade
+  add_foreign_key "lfs_file_locks", "users", on_delete: :cascade
   add_foreign_key "lists", "boards", name: "fk_0d3f677137", on_delete: :cascade
   add_foreign_key "lists", "labels", name: "fk_7a5553d60f", on_delete: :cascade
   add_foreign_key "members", "users", name: "fk_2e88fb7ce9", on_delete: :cascade
@@ -2041,9 +2060,13 @@ ActiveRecord::Schema.define(version: 20180105101928) do
   add_foreign_key "system_note_metadata", "notes", name: "fk_d83a918cb1", on_delete: :cascade
   add_foreign_key "timelogs", "issues", name: "fk_timelogs_issues_issue_id", on_delete: :cascade
   add_foreign_key "timelogs", "merge_requests", name: "fk_timelogs_merge_requests_merge_request_id", on_delete: :cascade
+  add_foreign_key "todos", "notes", name: "fk_91d1f47b13", on_delete: :cascade
   add_foreign_key "todos", "projects", name: "fk_45054f9c45", on_delete: :cascade
+  add_foreign_key "todos", "users", column: "author_id", name: "fk_ccf0373936", on_delete: :cascade
+  add_foreign_key "todos", "users", name: "fk_d94154aa95", on_delete: :cascade
   add_foreign_key "trending_projects", "projects", on_delete: :cascade
   add_foreign_key "u2f_registrations", "users"
+  add_foreign_key "user_callouts", "users", on_delete: :cascade
   add_foreign_key "user_custom_attributes", "users", on_delete: :cascade
   add_foreign_key "user_synced_attributes_metadata", "users", on_delete: :cascade
   add_foreign_key "users_star_projects", "projects", name: "fk_22cd27ddfc", on_delete: :cascade