Merge remote-tracking branch 'upstream/master' into 1819-override-ce

* upstream/master: (621 commits)
  Add a note about GitLab QA page objects validator to docs
  Refactor dispatcher projects blame and blob path
  Update export message to mention we can download the file from the UI
  Fix Ctrl+Enter keyboard shortcut saving comment/note edit
  fix case where tooltip messes up :last-child selector
  Add reason to keep postgresql 9.2 for CI
  Remove warning noise in ProjectImportOptions
  Add changelog entry
  Add RedirectRoute factory
  Update Ingress extra cost note to be more generic
  Fix Rubocop offense
  Refactor dispatcher project branches path
  Revert "Revert "Fix Route validation for unchanged path""
  Document that we need rsync for backing up
  Docs: move article "Laravel and Envoy w/ CI/CD"
  Recommend against the use of EFS
  Adds Rubocop rule for line break around conditionals
  Update CHANGELOG.md for 10.1.6
  Filter out build traces from logged parameters
  Refactored project:n* imports in dispatcher.js
  ...

25 files changed, 780 insertions, 44 deletions

diff --git a/db/fixtures/development/04_project.rb b/db/fixtures/development/04_project.rb
index 1f8f5cfc82b..213c8bca639 100644
--- a/db/fixtures/development/04_project.rb
+++ b/db/fixtures/development/04_project.rb
@@ -63,7 +63,8 @@ Sidekiq::Testing.inline! do
         namespace_id: group.id,
         name: project_path.titleize,
         description: FFaker::Lorem.sentence,
-        visibility_level: Gitlab::VisibilityLevel.values.sample
+        visibility_level: Gitlab::VisibilityLevel.values.sample,
+        skip_disk_validation: true
       }
 
       project = Projects::CreateService.new(User.first, params).execute
diff --git a/db/fixtures/development/06_teams.rb b/db/fixtures/development/06_teams.rb
index 8afb0dde077..b218f4e71fd 100644
--- a/db/fixtures/development/06_teams.rb
+++ b/db/fixtures/development/06_teams.rb
@@ -14,7 +14,7 @@ Sidekiq::Testing.inline! do
 
     Project.all.each do |project|
       User.all.sample(4).each do |user|
-        if project.add_role(user, Gitlab::Access.values.sample)
+        if project.add_role(user, Gitlab::Access.sym_options.keys.sample)
           print '.'
         else
           print 'F'
diff --git a/db/migrate/20160301174731_add_fingerprint_index.rb b/db/migrate/20160301174731_add_fingerprint_index.rb
new file mode 100644
index 00000000000..f2c3d1ba1ea
--- /dev/null
+++ b/db/migrate/20160301174731_add_fingerprint_index.rb
@@ -0,0 +1,17 @@
+# rubocop:disable all
+class AddFingerprintIndex < ActiveRecord::Migration
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  # https://gitlab.com/gitlab-org/gitlab-ee/issues/764
+  def change
+    args = [:keys, :fingerprint]
+
+    if Gitlab::Database.postgresql?
+      args << { algorithm: :concurrently }
+    end
+
+    add_index(*args) unless index_exists?(:keys, :fingerprint)
+  end
+end
diff --git a/db/migrate/20160621123729_add_rebase_commit_sha_to_merge_requests.rb b/db/migrate/20160621123729_add_rebase_commit_sha_to_merge_requests.rb
new file mode 100644
index 00000000000..1222dc640a8
--- /dev/null
+++ b/db/migrate/20160621123729_add_rebase_commit_sha_to_merge_requests.rb
@@ -0,0 +1,22 @@
+# This migration is a duplicate of 20171230123729_add_rebase_commit_sha_to_merge_requests_ce.rb
+#
+# We backported this feature from EE using the same migration, but with a new
+# timestamp, which caused an error when the backport was then to be merged back
+# into EE.
+#
+# See discussion at https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3932
+class AddRebaseCommitShaToMergeRequests < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    unless column_exists?(:merge_requests, :rebase_commit_sha)
+      add_column :merge_requests, :rebase_commit_sha, :string
+    end
+  end
+
+  def down
+    if column_exists?(:merge_requests, :rebase_commit_sha)
+      remove_column :merge_requests, :rebase_commit_sha
+    end
+  end
+end
diff --git a/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb b/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb
new file mode 100644
index 00000000000..1d86a531eb3
--- /dev/null
+++ b/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb
@@ -0,0 +1,19 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddAuthorizedKeysEnabledToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :application_settings, :authorized_keys_enabled, :boolean, default: true, allow_null: false
+  end
+
+  def down
+    remove_column :application_settings, :authorized_keys_enabled
+  end
+end
diff --git a/db/migrate/20170928124105_create_fork_networks.rb b/db/migrate/20170928124105_create_fork_networks.rb
index ca906b953a3..89e5b871967 100644
--- a/db/migrate/20170928124105_create_fork_networks.rb
+++ b/db/migrate/20170928124105_create_fork_networks.rb
@@ -23,6 +23,7 @@ class CreateForkNetworks < ActiveRecord::Migration
     if foreign_keys_for(:fork_networks, :root_project_id).any?
       remove_foreign_key :fork_networks, column: :root_project_id
     end
+
     drop_table :fork_networks
   end
 end
diff --git a/db/migrate/20170928133643_create_fork_network_members.rb b/db/migrate/20170928133643_create_fork_network_members.rb
index 836f023efdc..8c7d9ba859a 100644
--- a/db/migrate/20170928133643_create_fork_network_members.rb
+++ b/db/migrate/20170928133643_create_fork_network_members.rb
@@ -21,6 +21,7 @@ class CreateForkNetworkMembers < ActiveRecord::Migration
     if foreign_keys_for(:fork_network_members, :forked_from_project_id).any?
       remove_foreign_key :fork_network_members, column: :forked_from_project_id
     end
+
     drop_table :fork_network_members
   end
 end
diff --git a/db/migrate/20171019141859_fix_dev_timezone_schema.rb b/db/migrate/20171019141859_fix_dev_timezone_schema.rb
new file mode 100644
index 00000000000..fb7c17dd747
--- /dev/null
+++ b/db/migrate/20171019141859_fix_dev_timezone_schema.rb
@@ -0,0 +1,25 @@
+class FixDevTimezoneSchema < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # The this migrations tries to help solve unwanted changes to `schema.rb`
+  # while developing GitLab. Installations created before we started using
+  # `datetime_with_timezone` are likely to face this problem. Updating those
+  # columns to the new type should help fix this.
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  TIMEZONE_TABLES = %i(appearances ci_group_variables ci_pipeline_schedule_variables events gpg_keys gpg_signatures project_auto_devops)
+
+  def up
+    return unless Rails.env.development? || Rails.env.test?
+
+    TIMEZONE_TABLES.each do |table|
+      change_column table, :created_at, :datetime_with_timezone
+      change_column table, :updated_at, :datetime_with_timezone
+    end
+  end
+
+  def down
+  end
+end
diff --git a/db/migrate/20171106151218_issues_moved_to_id_foreign_key.rb b/db/migrate/20171106151218_issues_moved_to_id_foreign_key.rb
index 8d2ceb8cc18..6395462384b 100644
--- a/db/migrate/20171106151218_issues_moved_to_id_foreign_key.rb
+++ b/db/migrate/20171106151218_issues_moved_to_id_foreign_key.rb
@@ -15,8 +15,20 @@ class IssuesMovedToIdForeignKey < ActiveRecord::Migration
     self.table_name = 'issues'
 
     def self.with_orphaned_moved_to_issues
-      where('NOT EXISTS (SELECT true FROM issues WHERE issues.id = issues.moved_to_id)')
-        .where('moved_to_id IS NOT NULL')
+      if Gitlab::Database.postgresql?
+        # Be careful to use a second table here for comparison otherwise we'll null
+        # out all rows that don't have id == moved.to_id!
+        where('NOT EXISTS (SELECT true FROM issues B WHERE issues.moved_to_id = B.id)')
+          .where('moved_to_id IS NOT NULL')
+      else
+        # MySQL doesn't allow modification of the same table in a subquery,
+        # and using a temporary table isn't automatically guaranteed to work
+        # due to the MySQL query optimizer. See
+        # https://dev.mysql.com/doc/refman/5.7/en/update.html for more
+        # details.
+        joins('LEFT JOIN issues AS b ON issues.moved_to_id = b.id')
+          .where('issues.moved_to_id IS NOT NULL AND b.id IS NULL')
+      end
     end
   end
 
diff --git a/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb b/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb
new file mode 100644
index 00000000000..18af697cf88
--- /dev/null
+++ b/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb
@@ -0,0 +1,37 @@
+class AddEventsRelatedColumnsToMergeRequestMetrics < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    change_table :merge_request_metrics do |t|
+      t.references :merged_by, references: :users
+      t.references :latest_closed_by, references: :users
+    end
+
+    add_column :merge_request_metrics, :latest_closed_at, :datetime_with_timezone
+
+    add_concurrent_foreign_key :merge_request_metrics, :users,
+                               column: :merged_by_id,
+                               on_delete: :nullify
+
+    add_concurrent_foreign_key :merge_request_metrics, :users,
+                               column: :latest_closed_by_id,
+                               on_delete: :nullify
+  end
+
+  def down
+    if foreign_keys_for(:merge_request_metrics, :merged_by_id).any?
+      remove_foreign_key :merge_request_metrics, column: :merged_by_id
+    end
+
+    if foreign_keys_for(:merge_request_metrics, :latest_closed_by_id).any?
+      remove_foreign_key :merge_request_metrics, column: :latest_closed_by_id
+    end
+
+    remove_columns :merge_request_metrics,
+      :merged_by_id, :latest_closed_by_id, :latest_closed_at
+  end
+end
diff --git a/db/migrate/20171216111734_clean_up_for_members.rb b/db/migrate/20171216111734_clean_up_for_members.rb
new file mode 100644
index 00000000000..22e0997dce6
--- /dev/null
+++ b/db/migrate/20171216111734_clean_up_for_members.rb
@@ -0,0 +1,31 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CleanUpForMembers < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class Member < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'members'
+  end
+
+  def up
+    condition = <<~EOF.squish
+      invite_token IS NULL AND
+      NOT EXISTS (SELECT 1 FROM users WHERE users.id = members.user_id)
+    EOF
+
+    Member.each_batch(of: 10_000) do |batch|
+      batch.where(condition).delete_all
+    end
+  end
+
+  def down
+  end
+end
diff --git a/db/migrate/20171216112339_add_foreign_key_for_members.rb b/db/migrate/20171216112339_add_foreign_key_for_members.rb
new file mode 100644
index 00000000000..be17769be6a
--- /dev/null
+++ b/db/migrate/20171216112339_add_foreign_key_for_members.rb
@@ -0,0 +1,21 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddForeignKeyForMembers < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key(:members,
+                               :users,
+                               column: :user_id)
+  end
+
+  def down
+    remove_foreign_key(:members, column: :user_id)
+  end
+end
diff --git a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb
index 7cf1d0cec68..d1a039ed551 100644
--- a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb
+++ b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb
@@ -9,6 +9,7 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration
     return unless Gitlab::Database.postgresql?
 
     disable_statement_timeout
+
     if Gitlab::Database.version.to_f >= 9.5
       # Allow us to hot-patch the index manually ahead of the migration
       execute "CREATE INDEX CONCURRENTLY IF NOT EXISTS #{INDEX_NAME} ON namespaces (lower(name));"
@@ -21,6 +22,7 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration
     return unless Gitlab::Database.postgresql?
 
     disable_statement_timeout
+
     if Gitlab::Database.version.to_f >= 9.2
       execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME};"
     else
diff --git a/db/migrate/20171222183504_add_jobs_cache_index_to_project.rb b/db/migrate/20171222183504_add_jobs_cache_index_to_project.rb
new file mode 100644
index 00000000000..607e9d027d7
--- /dev/null
+++ b/db/migrate/20171222183504_add_jobs_cache_index_to_project.rb
@@ -0,0 +1,13 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddJobsCacheIndexToProject < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    add_column :projects, :jobs_cache_index, :integer
+  end
+end
diff --git a/db/migrate/20171229225929_change_user_project_limit_not_null_and_remove_default.rb b/db/migrate/20171229225929_change_user_project_limit_not_null_and_remove_default.rb
new file mode 100644
index 00000000000..54fbbcf1a0d
--- /dev/null
+++ b/db/migrate/20171229225929_change_user_project_limit_not_null_and_remove_default.rb
@@ -0,0 +1,38 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class ChangeUserProjectLimitNotNullAndRemoveDefault < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  # When a migration requires downtime you **must** uncomment the following
+  # constant and define a short and easy to understand explanation as to why the
+  # migration requires downtime.
+  # DOWNTIME_REASON = ''
+
+  # When using the methods "add_concurrent_index", "remove_concurrent_index" or
+  # "add_column_with_default" you must disable the use of transactions
+  # as these methods can not run in an existing transaction.
+  # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure
+  # that either of them is the _only_ method called in the migration,
+  # any other changes should go in a separate migration.
+  # This ensures that upon failure _only_ the index creation or removing fails
+  # and can be retried or reverted easily.
+  #
+  # To disable transactions uncomment the following line and remove these
+  # comments:
+  # disable_ddl_transaction!
+
+  def up
+    # Set Users#projects_limit to NOT NULL and remove the default value
+    change_column_null :users, :projects_limit, false
+    change_column_default :users, :projects_limit, nil
+  end
+
+  def down
+    change_column_null :users, :projects_limit, true
+    change_column_default :users, :projects_limit, 10
+  end
+end
diff --git a/db/migrate/20171230123729_add_rebase_commit_sha_to_merge_requests_ce.rb b/db/migrate/20171230123729_add_rebase_commit_sha_to_merge_requests_ce.rb
new file mode 100644
index 00000000000..94a7c1019d8
--- /dev/null
+++ b/db/migrate/20171230123729_add_rebase_commit_sha_to_merge_requests_ce.rb
@@ -0,0 +1,15 @@
+class AddRebaseCommitShaToMergeRequestsCe < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    unless column_exists?(:merge_requests, :rebase_commit_sha)
+      add_column :merge_requests, :rebase_commit_sha, :string
+    end
+  end
+
+  def down
+    if column_exists?(:merge_requests, :rebase_commit_sha)
+      remove_column :merge_requests, :rebase_commit_sha
+    end
+  end
+end
diff --git a/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb b/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb
new file mode 100644
index 00000000000..f942b4c062e
--- /dev/null
+++ b/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb
@@ -0,0 +1,29 @@
+class AddCommitsCountToMergeRequestDiff < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  MIGRATION = 'AddMergeRequestDiffCommitsCount'.freeze
+  BATCH_SIZE = 5000
+  DELAY_INTERVAL = 5.minutes.to_i
+
+  class MergeRequestDiff < ActiveRecord::Base
+    self.table_name = 'merge_request_diffs'
+
+    include ::EachBatch
+  end
+
+  disable_ddl_transaction!
+
+  def up
+    add_column :merge_request_diffs, :commits_count, :integer
+
+    say 'Populating the MergeRequestDiff `commits_count`'
+
+    queue_background_migration_jobs_by_range_at_intervals(MergeRequestDiff, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+  def down
+    remove_column :merge_request_diffs, :commits_count
+  end
+end
diff --git a/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb b/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb
index da0fcda87a6..17ad7de065d 100644
--- a/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb
+++ b/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb
@@ -31,6 +31,7 @@ class RenameUsersWithRenamedNamespace < ActiveRecord::Migration
       predicate = namespaces[:owner_id].eq(users[:id])
                     .and(namespaces[:type].eq(nil))
                     .and(users[:username].matches(path))
+
       update_sql = if Gitlab::Database.postgresql?
                      "UPDATE users SET username = namespaces.path "\
                      "FROM namespaces WHERE #{predicate.to_sql}"
diff --git a/db/post_migrate/20170907170235_delete_conflicting_redirect_routes.rb b/db/post_migrate/20170907170235_delete_conflicting_redirect_routes.rb
index 3e84b295be4..033019c398e 100644
--- a/db/post_migrate/20170907170235_delete_conflicting_redirect_routes.rb
+++ b/db/post_migrate/20170907170235_delete_conflicting_redirect_routes.rb
@@ -2,36 +2,12 @@
 # for more information on how to write migrations for GitLab.
 
 class DeleteConflictingRedirectRoutes < ActiveRecord::Migration
-  include Gitlab::Database::MigrationHelpers
-
-  DOWNTIME = false
-  MIGRATION = 'DeleteConflictingRedirectRoutesRange'.freeze
-  BATCH_SIZE = 200 # At 200, I expect under 20s per batch, which is under our query timeout of 60s.
-  DELAY_INTERVAL = 12.seconds
-
-  disable_ddl_transaction!
-
-  class Route < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'routes'
-  end
-
   def up
-    say opening_message
-
-    queue_background_migration_jobs_by_range_at_intervals(Route, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+    # No-op.
+    # See https://gitlab.com/gitlab-com/infrastructure/issues/3460#note_53223252
   end
 
   def down
     # nothing
   end
-
-  def opening_message
-    <<~MSG
-      Clean up redirect routes that conflict with regular routes.
-         See initial bug fix:
-         https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/13357
-    MSG
-  end
 end
diff --git a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb
new file mode 100644
index 00000000000..11b581e4b57
--- /dev/null
+++ b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb
@@ -0,0 +1,151 @@
+class MigrateKubernetesServiceToNewClustersArchitectures < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME = 'KubernetesService'.freeze
+
+  disable_ddl_transaction!
+
+  class Project < ActiveRecord::Base
+    self.table_name = 'projects'
+
+    has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject'
+    has_many :clusters, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+    has_many :services, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service'
+    has_one :kubernetes_service, -> { where(category: 'deployment', type: 'KubernetesService') }, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service', inverse_of: :project, foreign_key: :project_id
+  end
+
+  class Cluster < ActiveRecord::Base
+    self.table_name = 'clusters'
+
+    has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject'
+    has_many :projects, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project'
+    has_one :platform_kubernetes, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::PlatformsKubernetes'
+
+    accepts_nested_attributes_for :platform_kubernetes
+
+    enum platform_type: {
+      kubernetes: 1
+    }
+
+    enum provider_type: {
+      user: 0,
+      gcp: 1
+    }
+  end
+
+  class ClustersProject < ActiveRecord::Base
+    self.table_name = 'cluster_projects'
+
+    belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+    belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project'
+  end
+
+  class PlatformsKubernetes < ActiveRecord::Base
+    self.table_name = 'cluster_platforms_kubernetes'
+
+    belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+
+    attr_encrypted :token,
+      mode: :per_attribute_iv,
+      key: Gitlab::Application.secrets.db_key_base,
+      algorithm: 'aes-256-cbc'
+  end
+
+  class Service < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'services'
+    self.inheritance_column = :_type_disabled # Disable STI, otherwise KubernetesModel will be looked up
+
+    belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project', foreign_key: :project_id
+
+    scope :unmanaged_kubernetes_service, -> do
+      joins('LEFT JOIN projects ON projects.id = services.project_id')
+      .joins('LEFT JOIN cluster_projects ON cluster_projects.project_id = projects.id')
+      .joins('LEFT JOIN cluster_platforms_kubernetes ON cluster_platforms_kubernetes.cluster_id = cluster_projects.cluster_id')
+      .where(category: 'deployment', type: 'KubernetesService', template: false)
+      .where("services.properties LIKE '%api_url%'")
+      .where("(services.properties NOT LIKE CONCAT('%', cluster_platforms_kubernetes.api_url, '%')) OR cluster_platforms_kubernetes.api_url IS NULL")
+      .group(:id)
+      .order(id: :asc)
+    end
+
+    scope :kubernetes_service_without_template, -> do
+      where(category: 'deployment', type: 'KubernetesService', template: false)
+    end
+
+    def api_url
+      parsed_properties['api_url']
+    end
+
+    def ca_pem
+      parsed_properties['ca_pem']
+    end
+
+    def namespace
+      parsed_properties['namespace']
+    end
+
+    def token
+      parsed_properties['token']
+    end
+
+    private
+
+    def parsed_properties
+      @parsed_properties ||= JSON.parse(self.properties)
+    end
+  end
+
+  def find_dedicated_environement_scope(project)
+    environment_scopes = project.clusters.map(&:environment_scope)
+
+    return '*' if environment_scopes.exclude?('*') # KubernetesService should be added as a default cluster (environment_scope: '*') at first place
+    return 'migrated/*' if environment_scopes.exclude?('migrated/*') # If it's conflicted, the KubernetesService added as a migrated cluster
+
+    unique_iid = 0
+
+    # If it's still conflicted, finding an unique environment scope incrementaly
+    loop do
+      candidate = "migrated#{unique_iid}/*"
+      return candidate if environment_scopes.exclude?(candidate)
+
+      unique_iid += 1
+    end
+  end
+
+  def up
+    ActiveRecord::Base.transaction do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service
+        .unmanaged_kubernetes_service.find_each(batch_size: 1) do |kubernetes_service|
+        MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create(
+          enabled: kubernetes_service.active,
+          user_id: nil, # KubernetesService doesn't have
+          name: DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME,
+          provider_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.provider_types[:user],
+          platform_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.platform_types[:kubernetes],
+          projects: [kubernetes_service.project],
+          environment_scope: find_dedicated_environement_scope(kubernetes_service.project),
+          platform_kubernetes_attributes: {
+            api_url: kubernetes_service.api_url,
+            ca_cert: kubernetes_service.ca_pem,
+            namespace: kubernetes_service.namespace,
+            username: nil, # KubernetesService doesn't have
+            encrypted_password: nil, # KubernetesService doesn't have
+            encrypted_password_iv: nil, # KubernetesService doesn't have
+            token: kubernetes_service.token # encrypted_token and encrypted_token_iv
+          } )
+      end
+    end
+
+    MigrateKubernetesServiceToNewClustersArchitectures::Service
+      .kubernetes_service_without_template.each_batch(of: 100) do |kubernetes_service|
+      kubernetes_service.update_all(active: false)
+    end
+  end
+
+  def down
+    # noop
+  end
+end
diff --git a/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb
new file mode 100644
index 00000000000..fce1829c982
--- /dev/null
+++ b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+# rubocop:disable GitlabSecurity/SqlInjection
+
+class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migration
+  DOWNTIME = false
+  BATCH_SIZE = 10_000
+  MIGRATION = 'PopulateMergeRequestMetricsWithEventsData'
+
+  disable_ddl_transaction!
+
+  class MergeRequest < ActiveRecord::Base
+    self.table_name = 'merge_requests'
+
+    include ::EachBatch
+  end
+
+  def up
+    say 'Scheduling `PopulateMergeRequestMetricsWithEventsData` jobs'
+    # It will update around 4_000_000 records in batches of 10_000 merge
+    # requests (running between 10 minutes) and should take around 66 hours to complete.
+    # Apparently, production PostgreSQL is able to vacuum 10k-20k dead_tuples by
+    # minute, and at maximum, each of these jobs should UPDATE 20k records.
+    #
+    # More information about the updates in `PopulateMergeRequestMetricsWithEventsData` class.
+    #
+    MergeRequest.all.each_batch(of: BATCH_SIZE) do |relation, index|
+      range = relation.pluck('MIN(id)', 'MAX(id)').first
+
+      BackgroundMigrationWorker.perform_in(index * 10.minutes, MIGRATION, range)
+    end
+  end
+
+  def down
+    execute "update merge_request_metrics set latest_closed_at = null"
+    execute "update merge_request_metrics set latest_closed_by_id = null"
+    execute "update merge_request_metrics set merged_by_id = null"
+  end
+end
diff --git a/db/post_migrate/20171207150343_remove_soft_removed_objects.rb b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb
new file mode 100644
index 00000000000..3e2dedfdd6a
--- /dev/null
+++ b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb
@@ -0,0 +1,208 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveSoftRemovedObjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  module SoftRemoved
+    extend ActiveSupport::Concern
+
+    included do
+      scope :soft_removed, -> { where('deleted_at IS NOT NULL') }
+    end
+  end
+
+  class User < ActiveRecord::Base
+    self.table_name = 'users'
+
+    include EachBatch
+  end
+
+  class Issue < ActiveRecord::Base
+    self.table_name = 'issues'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class MergeRequest < ActiveRecord::Base
+    self.table_name = 'merge_requests'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class Namespace < ActiveRecord::Base
+    self.table_name = 'namespaces'
+
+    include EachBatch
+    include SoftRemoved
+
+    scope :soft_removed_personal, -> { soft_removed.where(type: nil) }
+    scope :soft_removed_group, -> { soft_removed.where(type: 'Group') }
+  end
+
+  class Route < ActiveRecord::Base
+    self.table_name = 'routes'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class Project < ActiveRecord::Base
+    self.table_name = 'projects'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class CiPipelineSchedule < ActiveRecord::Base
+    self.table_name = 'ci_pipeline_schedules'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class CiTrigger < ActiveRecord::Base
+    self.table_name = 'ci_triggers'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  MODELS = [Issue, MergeRequest, CiPipelineSchedule, CiTrigger].freeze
+
+  def up
+    disable_statement_timeout
+
+    remove_personal_routes
+    remove_personal_namespaces
+    remove_group_namespaces
+    remove_simple_soft_removed_rows
+  end
+
+  def down
+    # The data removed by this migration can't be restored in an automated way.
+  end
+
+  def remove_simple_soft_removed_rows
+    create_temporary_indexes
+
+    MODELS.each do |model|
+      say_with_time("Removing soft removed rows from #{model.table_name}") do
+        model.soft_removed.each_batch do |batch, index|
+          batch.delete_all
+        end
+      end
+    end
+  ensure
+    remove_temporary_indexes
+  end
+
+  def create_temporary_indexes
+    MODELS.each do |model|
+      index_name = temporary_index_name_for(model)
+
+      # Without this index the removal process can take a very long time. For
+      # example, getting the next ID of a batch for the `issues` table in
+      # staging would take between 15 and 20 seconds.
+      next if temporary_index_exists?(model)
+
+      say_with_time("Creating temporary index #{index_name}") do
+        add_concurrent_index(
+          model.table_name,
+          [:deleted_at, :id],
+          name: index_name,
+          where: 'deleted_at IS NOT NULL'
+        )
+      end
+    end
+  end
+
+  def remove_temporary_indexes
+    MODELS.each do |model|
+      index_name = temporary_index_name_for(model)
+
+      next unless temporary_index_exists?(model)
+
+      say_with_time("Removing temporary index #{index_name}") do
+        remove_concurrent_index_by_name(model.table_name, index_name)
+      end
+    end
+  end
+
+  def temporary_index_name_for(model)
+    "index_on_#{model.table_name}_tmp"
+  end
+
+  def temporary_index_exists?(model)
+    index_name = temporary_index_name_for(model)
+
+    index_exists?(model.table_name, [:deleted_at, :id], name: index_name)
+  end
+
+  def remove_personal_namespaces
+    # Some personal namespaces are left behind in case of GitLab.com. In these
+    # cases the associated data such as the projects and users has already been
+    # removed.
+    Namespace.soft_removed_personal.each_batch do |batch|
+      batch.delete_all
+    end
+  end
+
+  def remove_group_namespaces
+    admin_id = id_for_admin_user
+
+    unless admin_id
+      say 'Not scheduling soft removed groups for removal as no admin user ' \
+        'could be found. You will need to remove any such groups manually.'
+
+      return
+    end
+
+    # Left over groups can't be easily removed because we may also need to
+    # remove memberships, repositories, and other associated data. As a result
+    # we'll just schedule a Sidekiq job to remove these.
+    #
+    # As of January 5th, 2018 there are 36 groups that will be removed using
+    # this code.
+    Namespace.select(:id).soft_removed_group.each_batch(of: 10) do |batch, index|
+      batch.each do |ns|
+        schedule_group_removal(index * 5.minutes, ns.id, admin_id)
+      end
+    end
+  end
+
+  def schedule_group_removal(delay, group_id, user_id)
+    if migrate_inline?
+      GroupDestroyWorker.new.perform(group_id, user_id)
+    else
+      GroupDestroyWorker.perform_in(delay, group_id, user_id)
+    end
+  end
+
+  def remove_personal_routes
+    namespaces = Namespace.select(1)
+      .soft_removed
+      .where('namespaces.type IS NULL')
+      .where('routes.source_type = ?', 'Namespace')
+      .where('routes.source_id = namespaces.id')
+
+    Route.where('EXISTS (?)', namespaces).each_batch do |batch|
+      batch.delete_all
+    end
+  end
+
+  def id_for_admin_user
+    User.where(admin: true).limit(1).pluck(:id).first
+  end
+
+  def migrate_inline?
+    Rails.env.test? || Rails.env.development?
+  end
+end
diff --git a/db/post_migrate/20171207150344_remove_deleted_at_columns.rb b/db/post_migrate/20171207150344_remove_deleted_at_columns.rb
new file mode 100644
index 00000000000..154d7a1b926
--- /dev/null
+++ b/db/post_migrate/20171207150344_remove_deleted_at_columns.rb
@@ -0,0 +1,31 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveDeletedAtColumns < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  TABLES = %i[issues merge_requests namespaces ci_pipeline_schedules ci_triggers].freeze
+  COLUMN = :deleted_at
+
+  def up
+    TABLES.each do |table|
+      remove_column(table, COLUMN) if column_exists?(table, COLUMN)
+    end
+  end
+
+  def down
+    TABLES.each do |table|
+      unless column_exists?(table, COLUMN)
+        add_column(table, COLUMN, :datetime_with_timezone)
+      end
+
+      unless index_exists?(table, COLUMN)
+        add_concurrent_index(table, COLUMN)
+      end
+    end
+  end
+end
diff --git a/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb b/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb
new file mode 100644
index 00000000000..eeecc7b1de0
--- /dev/null
+++ b/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb
@@ -0,0 +1,45 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+# rubocop:disable Migration/Datetime
+class ScheduleIssuesClosedAtTypeChange < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class Issue < ActiveRecord::Base
+    self.table_name = 'issues'
+    include EachBatch
+
+    def self.to_migrate
+      where('closed_at IS NOT NULL')
+    end
+  end
+
+  def up
+    return unless migrate_column_type?
+
+    change_column_type_using_background_migration(
+      Issue.to_migrate,
+      :closed_at,
+      :datetime_with_timezone
+    )
+  end
+
+  def down
+    return if migrate_column_type?
+
+    change_column_type_using_background_migration(
+      Issue.to_migrate,
+      :closed_at,
+      :datetime
+    )
+  end
+
+  def migrate_column_type?
+    # Some environments may have already executed the previous version of this
+    # migration, thus we don't need to migrate those environments again.
+    column_for('issues', 'closed_at').type == :datetime
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index aa5db5da4f0..8a6db61250b 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171220191323) do
+ActiveRecord::Schema.define(version: 20180105212544) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -154,6 +154,7 @@ ActiveRecord::Schema.define(version: 20171220191323) do
     t.integer "gitaly_timeout_default", default: 55, null: false
     t.integer "gitaly_timeout_medium", default: 30, null: false
     t.integer "gitaly_timeout_fast", default: 10, null: false
+    t.boolean "authorized_keys_enabled", default: true, null: false
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -356,7 +357,6 @@ ActiveRecord::Schema.define(version: 20171220191323) do
     t.integer "project_id"
     t.integer "owner_id"
     t.boolean "active", default: true
-    t.datetime "deleted_at"
     t.datetime "created_at"
     t.datetime "updated_at"
   end
@@ -466,7 +466,6 @@ ActiveRecord::Schema.define(version: 20171220191323) do
 
   create_table "ci_triggers", force: :cascade do |t|
     t.string "token"
-    t.datetime "deleted_at"
     t.datetime "created_at"
     t.datetime "updated_at"
     t.integer "project_id"
@@ -657,8 +656,8 @@ ActiveRecord::Schema.define(version: 20171220191323) do
     t.datetime "created_at"
     t.datetime "updated_at"
     t.string "confirmation_token"
-    t.datetime "confirmed_at"
-    t.datetime "confirmation_sent_at"
+    t.datetime_with_timezone "confirmed_at"
+    t.datetime_with_timezone "confirmation_sent_at"
   end
 
   add_index "emails", ["confirmation_token"], name: "index_emails_on_confirmation_token", unique: true, using: :btree
@@ -860,7 +859,6 @@ ActiveRecord::Schema.define(version: 20171220191323) do
     t.integer "iid"
     t.integer "updated_by_id"
     t.boolean "confidential", default: false, null: false
-    t.datetime "deleted_at"
     t.date "due_date"
     t.integer "moved_to_id"
     t.integer "lock_version"
@@ -877,7 +875,6 @@ ActiveRecord::Schema.define(version: 20171220191323) do
 
   add_index "issues", ["author_id"], name: "index_issues_on_author_id", using: :btree
   add_index "issues", ["confidential"], name: "index_issues_on_confidential", using: :btree
-  add_index "issues", ["deleted_at"], name: "index_issues_on_deleted_at", using: :btree
   add_index "issues", ["description"], name: "index_issues_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
   add_index "issues", ["milestone_id"], name: "index_issues_on_milestone_id", using: :btree
   add_index "issues", ["moved_to_id"], name: "index_issues_on_moved_to_id", where: "(moved_to_id IS NOT NULL)", using: :btree
@@ -1043,6 +1040,7 @@ ActiveRecord::Schema.define(version: 20171220191323) do
     t.string "real_size"
     t.string "head_commit_sha"
     t.string "start_commit_sha"
+    t.integer "commits_count"
   end
 
   add_index "merge_request_diffs", ["merge_request_id", "id"], name: "index_merge_request_diffs_on_merge_request_id_and_id", using: :btree
@@ -1056,6 +1054,9 @@ ActiveRecord::Schema.define(version: 20171220191323) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.integer "pipeline_id"
+    t.integer "merged_by_id"
+    t.integer "latest_closed_by_id"
+    t.datetime_with_timezone "latest_closed_at"
   end
 
   add_index "merge_request_metrics", ["first_deployed_to_production_at"], name: "index_merge_request_metrics_on_first_deployed_to_production_at", using: :btree
@@ -1083,7 +1084,6 @@ ActiveRecord::Schema.define(version: 20171220191323) do
     t.boolean "merge_when_pipeline_succeeds", default: false, null: false
     t.integer "merge_user_id"
     t.string "merge_commit_sha"
-    t.datetime "deleted_at"
     t.string "in_progress_merge_commit_sha"
     t.integer "lock_version"
     t.text "title_html"
@@ -1096,12 +1096,12 @@ ActiveRecord::Schema.define(version: 20171220191323) do
     t.string "merge_jid"
     t.boolean "discussion_locked"
     t.integer "latest_merge_request_diff_id"
+    t.string "rebase_commit_sha"
   end
 
   add_index "merge_requests", ["assignee_id"], name: "index_merge_requests_on_assignee_id", using: :btree
   add_index "merge_requests", ["author_id"], name: "index_merge_requests_on_author_id", using: :btree
   add_index "merge_requests", ["created_at"], name: "index_merge_requests_on_created_at", using: :btree
-  add_index "merge_requests", ["deleted_at"], name: "index_merge_requests_on_deleted_at", using: :btree
   add_index "merge_requests", ["description"], name: "index_merge_requests_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
   add_index "merge_requests", ["head_pipeline_id"], name: "index_merge_requests_on_head_pipeline_id", using: :btree
   add_index "merge_requests", ["latest_merge_request_diff_id"], name: "index_merge_requests_on_latest_merge_request_diff_id", using: :btree
@@ -1161,7 +1161,6 @@ ActiveRecord::Schema.define(version: 20171220191323) do
     t.boolean "share_with_group_lock", default: false
     t.integer "visibility_level", default: 20, null: false
     t.boolean "request_access_enabled", default: false, null: false
-    t.datetime "deleted_at"
     t.text "description_html"
     t.boolean "lfs_enabled"
     t.integer "parent_id"
@@ -1171,7 +1170,6 @@ ActiveRecord::Schema.define(version: 20171220191323) do
   end
 
   add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree
-  add_index "namespaces", ["deleted_at"], name: "index_namespaces_on_deleted_at", using: :btree
   add_index "namespaces", ["name", "parent_id"], name: "index_namespaces_on_name_and_parent_id", unique: true, using: :btree
   add_index "namespaces", ["name"], name: "index_namespaces_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"}
   add_index "namespaces", ["owner_id"], name: "index_namespaces_on_owner_id", using: :btree
@@ -1447,6 +1445,7 @@ ActiveRecord::Schema.define(version: 20171220191323) do
     t.boolean "repository_read_only"
     t.boolean "merge_requests_ff_only_enabled", default: false
     t.boolean "merge_requests_rebase_enabled", default: false, null: false
+    t.integer "jobs_cache_index"
   end
 
   add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
@@ -1770,8 +1769,8 @@ ActiveRecord::Schema.define(version: 20171220191323) do
   add_index "user_agent_details", ["subject_id", "subject_type"], name: "index_user_agent_details_on_subject_id_and_subject_type", using: :btree
 
   create_table "user_custom_attributes", force: :cascade do |t|
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
     t.integer "user_id", null: false
     t.string "key", null: false
     t.string "value", null: false
@@ -1805,7 +1804,7 @@ ActiveRecord::Schema.define(version: 20171220191323) do
     t.datetime "updated_at"
     t.string "name"
     t.boolean "admin", default: false, null: false
-    t.integer "projects_limit", default: 10
+    t.integer "projects_limit", null: false
     t.string "skype", default: "", null: false
     t.string "linkedin", default: "", null: false
     t.string "twitter", default: "", null: false
@@ -1989,11 +1988,14 @@ ActiveRecord::Schema.define(version: 20171220191323) do
   add_foreign_key "labels", "projects", name: "fk_7de4989a69", on_delete: :cascade
   add_foreign_key "lists", "boards", name: "fk_0d3f677137", on_delete: :cascade
   add_foreign_key "lists", "labels", name: "fk_7a5553d60f", on_delete: :cascade
+  add_foreign_key "members", "users", name: "fk_2e88fb7ce9", on_delete: :cascade
   add_foreign_key "merge_request_diff_commits", "merge_request_diffs", on_delete: :cascade
   add_foreign_key "merge_request_diff_files", "merge_request_diffs", on_delete: :cascade
   add_foreign_key "merge_request_diffs", "merge_requests", name: "fk_8483f3258f", on_delete: :cascade
   add_foreign_key "merge_request_metrics", "ci_pipelines", column: "pipeline_id", on_delete: :cascade
   add_foreign_key "merge_request_metrics", "merge_requests", on_delete: :cascade
+  add_foreign_key "merge_request_metrics", "users", column: "latest_closed_by_id", name: "fk_ae440388cc", on_delete: :nullify
+  add_foreign_key "merge_request_metrics", "users", column: "merged_by_id", name: "fk_7f28d925f3", on_delete: :nullify
   add_foreign_key "merge_requests", "ci_pipelines", column: "head_pipeline_id", name: "fk_fd82eae0b9", on_delete: :nullify
   add_foreign_key "merge_requests", "merge_request_diffs", column: "latest_merge_request_diff_id", name: "fk_06067f5644", on_delete: :nullify
   add_foreign_key "merge_requests", "milestones", name: "fk_6a5165a692", on_delete: :nullify