Merge branch 'master' into 44427-state-management-with-vuext

* master: (1063 commits)
  Replace commits spinach tests with RSpec analog
  Update repository.rb
  Add note about rebase/squash duplication in Gitaly
  Resolve "Reconcile project templates with Auto DevOps"
  Move import project pane to a separate partial
  Inform the user when there are no project import options available
  Clarify location of Vue templates
  Make add_index_to_namespaces_runners_token migration reversible
  Fix lambda arguments in Grape entities
  Update grape-entity 0.6.0 -> 0.7.1
  Fix constants in backfill_runner_type_for_ci_runners_post_migrate.rb
  Use limited_counter_with_delimiter in the admin user list tabs
  Remove a warning from spec/features/admin/admin_users_spec.rb
  Use smallint for runner_type since its an enum
  Dont remove duplicates in Runner.owned_or_shared since its not necessary
  Change the docs license to CC BY-SA
  Remove unnecessary disable transaction in add_ci_runner_namespaces
  Split migration to add and index namespaces.runners_token
  Output some useful information when running the rails console
  Revert "Use factory in specs for ProjectCiCdSettings"
  ...

31 files changed, 651 insertions, 54 deletions

diff --git a/db/fixtures/development/01_admin.rb b/db/fixtures/development/01_admin.rb
index dfb50c195c1..1e260236dc5 100644
--- a/db/fixtures/development/01_admin.rb
+++ b/db/fixtures/development/01_admin.rb
@@ -1,14 +1,14 @@
 require './spec/support/sidekiq'
 
 Gitlab::Seeder.quiet do
-  User.seed do |s|
-    s.id = 1
-    s.name = 'Administrator'
-    s.email = 'admin@example.com'
-    s.notification_email = 'admin@example.com'
-    s.username = 'root'
-    s.password = '5iveL!fe'
-    s.admin = true
-    s.confirmed_at = DateTime.now
-  end
+  User.create!(
+    name: 'Administrator',
+    email: 'admin@example.com',
+    username: 'root',
+    password: '5iveL!fe',
+    admin: true,
+    confirmed_at: DateTime.now
+  )
+
+  print '.'
 end
diff --git a/db/fixtures/development/10_merge_requests.rb b/db/fixtures/development/10_merge_requests.rb
index 30244ee4431..bcfdd058a1c 100644
--- a/db/fixtures/development/10_merge_requests.rb
+++ b/db/fixtures/development/10_merge_requests.rb
@@ -4,7 +4,7 @@ Gitlab::Seeder.quiet do
   # Limit the number of merge requests per project to avoid long seeds
   MAX_NUM_MERGE_REQUESTS = 10
 
-  Project.all.reject(&:empty_repo?).each do |project|
+  Project.non_archived.with_merge_requests_enabled.reject(&:empty_repo?).each do |project|
     branches = project.repository.branch_names.sample(MAX_NUM_MERGE_REQUESTS * 2)
 
     branches.each do |branch_name|
@@ -21,7 +21,11 @@ Gitlab::Seeder.quiet do
         assignee: project.team.users.sample
       }
 
-      MergeRequests::CreateService.new(project, project.team.users.sample, params).execute
+      # Only create MRs with users that are allowed to create MRs
+      developer = project.team.developers.sample
+      break unless developer
+
+      MergeRequests::CreateService.new(project, developer, params).execute
       print '.'
     end
   end
diff --git a/db/fixtures/development/17_cycle_analytics.rb b/db/fixtures/development/17_cycle_analytics.rb
index d7be6f5950f..7b9a4bad449 100644
--- a/db/fixtures/development/17_cycle_analytics.rb
+++ b/db/fixtures/development/17_cycle_analytics.rb
@@ -1,5 +1,5 @@
 require './spec/support/sidekiq'
-require './spec/support/test_env'
+require './spec/support/helpers/test_env'
 
 class Gitlab::Seeder::CycleAnalytics
   def initialize(project, perf: false)
diff --git a/db/fixtures/development/19_environments.rb b/db/fixtures/development/19_environments.rb
index c1bbc9af6d6..00a14f458d1 100644
--- a/db/fixtures/development/19_environments.rb
+++ b/db/fixtures/development/19_environments.rb
@@ -28,7 +28,11 @@ class Gitlab::Seeder::Environments
   end
 
   def create_merge_request_review_deployments!
-    @project.merge_requests.sample(4).map do |merge_request|
+    @project
+      .merge_requests
+      .select { |mr| mr.source_branch.match(/\p{Alnum}+/) }
+      .sample(4)
+      .each do |merge_request|
       next unless merge_request.diff_head_sha
 
       create_deployment!(
diff --git a/db/migrate/20161220141214_remove_dot_git_from_group_names.rb b/db/migrate/20161220141214_remove_dot_git_from_group_names.rb
index bddc234db25..17357b67ab7 100644
--- a/db/migrate/20161220141214_remove_dot_git_from_group_names.rb
+++ b/db/migrate/20161220141214_remove_dot_git_from_group_names.rb
@@ -59,17 +59,17 @@ class RemoveDotGitFromGroupNames < ActiveRecord::Migration
   end
 
   def move_namespace(group_id, path_was, path)
-    repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row|
-      Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path
+    repository_storages = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row|
+      row['repository_storage']
     end.compact
 
     # Move the namespace directory in all storages paths used by member projects
-    repository_storage_paths.each do |repository_storage_path|
+    repository_storages.each do |repository_storage|
       # Ensure old directory exists before moving it
-      gitlab_shell.add_namespace(repository_storage_path, path_was)
+      gitlab_shell.add_namespace(repository_storage, path_was)
 
-      unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
-        Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}"
+      unless gitlab_shell.mv_namespace(repository_storage, path_was, path)
+        Rails.logger.error "Exception moving on shard #{repository_storage} from #{path_was} to #{path}"
 
         # if we cannot move namespace directory we should rollback
         # db changes in order to prevent out of sync between db and fs
diff --git a/db/migrate/20161226122833_remove_dot_git_from_usernames.rb b/db/migrate/20161226122833_remove_dot_git_from_usernames.rb
index 7c28d934c29..8986cd8cb4b 100644
--- a/db/migrate/20161226122833_remove_dot_git_from_usernames.rb
+++ b/db/migrate/20161226122833_remove_dot_git_from_usernames.rb
@@ -53,8 +53,8 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
     select_all("SELECT id, path FROM routes WHERE path = '#{quote_string(path)}'").present?
   end
 
-  def path_exists?(path, repository_storage_path)
-    repository_storage_path && gitlab_shell.exists?(repository_storage_path, path)
+  def path_exists?(shard, repository_storage_path)
+    repository_storage_path && gitlab_shell.exists?(shard, repository_storage_path)
   end
 
   # Accepts invalid path like test.git and returns test_git or
@@ -70,8 +70,8 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
   def check_routes(base, counter, path)
     route_exists = route_exists?(path)
 
-    Gitlab.config.repositories.storages.each_value do |storage|
-      if route_exists || path_exists?(path, storage.legacy_disk_path)
+    Gitlab.config.repositories.storages.each do |shard, storage|
+      if route_exists || path_exists?(shard, storage.legacy_disk_path)
         counter += 1
         path = "#{base}#{counter}"
 
@@ -83,17 +83,17 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
   end
 
   def move_namespace(namespace_id, path_was, path)
-    repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row|
-      Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path
+    repository_storages = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row|
+      row['repository_storage']
     end.compact
 
-    # Move the namespace directory in all storages paths used by member projects
-    repository_storage_paths.each do |repository_storage_path|
+    # Move the namespace directory in all storages used by member projects
+    repository_storages.each do |repository_storage|
       # Ensure old directory exists before moving it
-      gitlab_shell.add_namespace(repository_storage_path, path_was)
+      gitlab_shell.add_namespace(repository_storage, path_was)
 
-      unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
-        Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}"
+      unless gitlab_shell.mv_namespace(repository_storage, path_was, path)
+        Rails.logger.error "Exception moving on shard #{repository_storage} from #{path_was} to #{path}"
 
         # if we cannot move namespace directory we should rollback
         # db changes in order to prevent out of sync between db and fs
diff --git a/db/migrate/20170301101006_add_ci_runner_namespaces.rb b/db/migrate/20170301101006_add_ci_runner_namespaces.rb
new file mode 100644
index 00000000000..deaf03e928b
--- /dev/null
+++ b/db/migrate/20170301101006_add_ci_runner_namespaces.rb
@@ -0,0 +1,17 @@
+class AddCiRunnerNamespaces < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :ci_runner_namespaces do |t|
+      t.integer :runner_id
+      t.integer :namespace_id
+
+      t.index [:runner_id, :namespace_id], unique: true
+      t.index :namespace_id
+      t.foreign_key :ci_runners, column: :runner_id, on_delete: :cascade
+      t.foreign_key :namespaces, column: :namespace_id, on_delete: :cascade
+    end
+  end
+end
diff --git a/db/migrate/20170906133745_add_runners_token_to_groups.rb b/db/migrate/20170906133745_add_runners_token_to_groups.rb
new file mode 100644
index 00000000000..852f4cba670
--- /dev/null
+++ b/db/migrate/20170906133745_add_runners_token_to_groups.rb
@@ -0,0 +1,9 @@
+class AddRunnersTokenToGroups < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :namespaces, :runners_token, :string
+  end
+end
diff --git a/db/migrate/20171222115326_add_confidential_note_events_to_web_hooks.rb b/db/migrate/20171222115326_add_confidential_note_events_to_web_hooks.rb
new file mode 100644
index 00000000000..900a6386922
--- /dev/null
+++ b/db/migrate/20171222115326_add_confidential_note_events_to_web_hooks.rb
@@ -0,0 +1,15 @@
+class AddConfidentialNoteEventsToWebHooks < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column :web_hooks, :confidential_note_events, :boolean
+  end
+
+  def down
+    remove_column :web_hooks, :confidential_note_events
+  end
+end
diff --git a/db/migrate/20180103123548_add_confidential_note_events_to_services.rb b/db/migrate/20180103123548_add_confidential_note_events_to_services.rb
new file mode 100644
index 00000000000..b54ad88df43
--- /dev/null
+++ b/db/migrate/20180103123548_add_confidential_note_events_to_services.rb
@@ -0,0 +1,16 @@
+class AddConfidentialNoteEventsToServices < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column :services, :confidential_note_events, :boolean
+    change_column_default :services, :confidential_note_events, true
+  end
+
+  def down
+    remove_column :services, :confidential_note_events
+  end
+end
diff --git a/db/migrate/20180319190020_create_deploy_tokens.rb b/db/migrate/20180319190020_create_deploy_tokens.rb
new file mode 100644
index 00000000000..d129459ea0a
--- /dev/null
+++ b/db/migrate/20180319190020_create_deploy_tokens.rb
@@ -0,0 +1,19 @@
+class CreateDeployTokens < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    create_table :deploy_tokens do |t|
+      t.boolean :revoked, default: false
+      t.boolean :read_repository, null: false, default: false
+      t.boolean :read_registry, null: false, default: false
+
+      t.datetime_with_timezone :expires_at, null: false
+      t.datetime_with_timezone :created_at, null: false
+
+      t.string :name, null: false
+      t.string :token, index: { unique: true }, null: false
+
+      t.index [:token, :expires_at, :id], where: "(revoked IS FALSE)"
+    end
+  end
+end
diff --git a/db/migrate/20180330121048_add_issue_due_to_notification_settings.rb b/db/migrate/20180330121048_add_issue_due_to_notification_settings.rb
new file mode 100644
index 00000000000..c64a481fcf0
--- /dev/null
+++ b/db/migrate/20180330121048_add_issue_due_to_notification_settings.rb
@@ -0,0 +1,9 @@
+class AddIssueDueToNotificationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :notification_settings, :issue_due, :boolean
+  end
+end
diff --git a/db/migrate/20180403035759_create_project_ci_cd_settings.rb b/db/migrate/20180403035759_create_project_ci_cd_settings.rb
new file mode 100644
index 00000000000..06856af6204
--- /dev/null
+++ b/db/migrate/20180403035759_create_project_ci_cd_settings.rb
@@ -0,0 +1,68 @@
+class CreateProjectCiCdSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    unless table_exists?(:project_ci_cd_settings)
+      create_table(:project_ci_cd_settings) do |t|
+        t.integer(:project_id, null: false)
+        t.boolean(:group_runners_enabled, default: true, null: false)
+      end
+    end
+
+    disable_statement_timeout
+
+    # This particular INSERT will take between 10 and 20 seconds.
+    execute 'INSERT INTO project_ci_cd_settings (project_id) SELECT id FROM projects'
+
+    # We add the index and foreign key separately so the above INSERT statement
+    # takes as little time as possible.
+    add_concurrent_index(:project_ci_cd_settings, :project_id, unique: true)
+
+    add_foreign_key_with_retry
+  end
+
+  def down
+    drop_table :project_ci_cd_settings
+  end
+
+  def add_foreign_key_with_retry
+    if Gitlab::Database.mysql?
+      # When using MySQL we don't support online upgrades, thus projects can't
+      # be deleted while we are running this migration.
+      return add_project_id_foreign_key
+    end
+
+    # Between the initial INSERT and the addition of the foreign key some
+    # projects may have been removed, leaving orphaned rows in our new settings
+    # table.
+    loop do
+      remove_orphaned_settings
+
+      begin
+        add_project_id_foreign_key
+        break
+      rescue ActiveRecord::InvalidForeignKey
+        say 'project_ci_cd_settings contains some orphaned rows, retrying...'
+      end
+    end
+  end
+
+  def add_project_id_foreign_key
+    add_concurrent_foreign_key(:project_ci_cd_settings, :projects, column: :project_id)
+  end
+
+  def remove_orphaned_settings
+    execute <<~SQL
+      DELETE FROM project_ci_cd_settings
+      WHERE NOT EXISTS (
+        SELECT 1
+        FROM projects
+        WHERE projects.id = project_ci_cd_settings.project_id
+      )
+    SQL
+  end
+end
diff --git a/db/migrate/20180405142733_create_project_deploy_tokens.rb b/db/migrate/20180405142733_create_project_deploy_tokens.rb
new file mode 100644
index 00000000000..9d8f89243a8
--- /dev/null
+++ b/db/migrate/20180405142733_create_project_deploy_tokens.rb
@@ -0,0 +1,16 @@
+class CreateProjectDeployTokens < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    create_table :project_deploy_tokens do |t|
+      t.integer :project_id, null: false
+      t.integer :deploy_token_id, null: false
+      t.datetime_with_timezone :created_at, null: false
+
+      t.foreign_key :deploy_tokens, column: :deploy_token_id, on_delete: :cascade
+      t.foreign_key :projects, column: :project_id, on_delete: :cascade
+
+      t.index [:project_id, :deploy_token_id], unique: true
+    end
+  end
+end
diff --git a/db/migrate/20180413022611_create_missing_namespace_for_internal_users.rb b/db/migrate/20180413022611_create_missing_namespace_for_internal_users.rb
new file mode 100644
index 00000000000..8fc558be733
--- /dev/null
+++ b/db/migrate/20180413022611_create_missing_namespace_for_internal_users.rb
@@ -0,0 +1,66 @@
+class CreateMissingNamespaceForInternalUsers < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    connection.exec_query(users_query.to_sql).rows.each do |id, username|
+      create_namespace(id, username)
+      # When testing locally I've noticed that these internal users are missing
+      # the notification email, for more details visit the below link:
+      # https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18357#note_68327560
+      set_notification_email(id)
+    end
+  end
+
+  def down
+    # no-op
+  end
+
+  private
+
+  def users
+    @users ||= Arel::Table.new(:users)
+  end
+
+  def namespaces
+    @namespaces ||= Arel::Table.new(:namespaces)
+  end
+
+  def users_query
+    condition = users[:ghost].eq(true)
+
+    if column_exists?(:users, :support_bot)
+      condition = condition.or(users[:support_bot].eq(true))
+    end
+
+    users.join(namespaces, Arel::Nodes::OuterJoin)
+      .on(namespaces[:type].eq(nil).and(namespaces[:owner_id].eq(users[:id])))
+      .where(namespaces[:owner_id].eq(nil))
+      .where(condition)
+      .project(users[:id], users[:username])
+  end
+
+  def create_namespace(user_id, username)
+    path = Uniquify.new.string(username) do |str|
+      query = "SELECT id FROM namespaces WHERE parent_id IS NULL AND path='#{str}' LIMIT 1"
+      connection.exec_query(query).present?
+    end
+
+    insert_query = "INSERT INTO namespaces(owner_id, path, name) VALUES(#{user_id}, '#{path}', '#{path}')"
+    namespace_id = connection.insert_sql(insert_query)
+
+    create_route(namespace_id)
+  end
+
+  def create_route(namespace_id)
+    return unless namespace_id
+
+    row = connection.exec_query("SELECT id, path FROM namespaces WHERE id=#{namespace_id}").first
+    id, path = row.values_at('id', 'path')
+
+    execute("INSERT INTO routes(source_id, source_type, path, name) VALUES(#{id}, 'Namespace', '#{path}', '#{path}')")
+  end
+
+  def set_notification_email(user_id)
+    execute "UPDATE users SET notification_email = email WHERE notification_email IS NULL AND id = #{user_id}"
+  end
+end
diff --git a/db/migrate/20180416155103_add_further_scope_columns_to_internal_id_table.rb b/db/migrate/20180416155103_add_further_scope_columns_to_internal_id_table.rb
new file mode 100644
index 00000000000..37e2d19e022
--- /dev/null
+++ b/db/migrate/20180416155103_add_further_scope_columns_to_internal_id_table.rb
@@ -0,0 +1,15 @@
+class AddFurtherScopeColumnsToInternalIdTable < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    change_column_null :internal_ids, :project_id, true
+    add_column :internal_ids, :namespace_id, :integer, null: true
+  end
+
+  def down
+    change_column_null :internal_ids, :project_id, false
+    remove_column :internal_ids, :namespace_id
+  end
+end
diff --git a/db/migrate/20180417090132_add_index_constraints_to_internal_id_table.rb b/db/migrate/20180417090132_add_index_constraints_to_internal_id_table.rb
new file mode 100644
index 00000000000..582b89a3948
--- /dev/null
+++ b/db/migrate/20180417090132_add_index_constraints_to_internal_id_table.rb
@@ -0,0 +1,40 @@
+class AddIndexConstraintsToInternalIdTable < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :internal_ids, [:usage, :namespace_id], unique: true, where: 'namespace_id IS NOT NULL'
+
+    replace_index(:internal_ids, [:usage, :project_id], name: 'index_internal_ids_on_usage_and_project_id') do
+      add_concurrent_index :internal_ids, [:usage, :project_id], unique: true, where: 'project_id IS NOT NULL'
+    end
+
+    add_concurrent_foreign_key :internal_ids, :namespaces, column: :namespace_id, on_delete: :cascade
+  end
+
+  def down
+    remove_concurrent_index :internal_ids, [:usage, :namespace_id]
+
+    replace_index(:internal_ids, [:usage, :project_id], name: 'index_internal_ids_on_usage_and_project_id') do
+      add_concurrent_index :internal_ids, [:usage, :project_id], unique: true
+    end
+
+    remove_foreign_key :internal_ids, column: :namespace_id
+  end
+
+  private
+  def replace_index(table, columns, name:)
+    temporary_name = "#{name}_old"
+
+    if index_exists?(table, columns, name: name)
+      rename_index table, name, temporary_name
+    end
+
+    yield
+
+    remove_concurrent_index_by_name table, temporary_name
+  end
+end
diff --git a/db/migrate/20180417101040_add_tmp_stage_priority_index_to_ci_builds.rb b/db/migrate/20180417101040_add_tmp_stage_priority_index_to_ci_builds.rb
new file mode 100644
index 00000000000..ee82c70ecf8
--- /dev/null
+++ b/db/migrate/20180417101040_add_tmp_stage_priority_index_to_ci_builds.rb
@@ -0,0 +1,16 @@
+class AddTmpStagePriorityIndexToCiBuilds < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(:ci_builds, [:stage_id, :stage_idx],
+                         where: 'stage_idx IS NOT NULL', name: 'tmp_build_stage_position_index')
+  end
+
+  def down
+    remove_concurrent_index_by_name(:ci_builds, 'tmp_build_stage_position_index')
+  end
+end
diff --git a/db/migrate/20180417101940_add_index_to_ci_stage.rb b/db/migrate/20180417101940_add_index_to_ci_stage.rb
new file mode 100644
index 00000000000..9dac78db774
--- /dev/null
+++ b/db/migrate/20180417101940_add_index_to_ci_stage.rb
@@ -0,0 +1,9 @@
+class AddIndexToCiStage < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_stages, :position, :integer
+  end
+end
diff --git a/db/migrate/20180418053107_add_index_to_ci_job_artifacts_file_store.rb b/db/migrate/20180418053107_add_index_to_ci_job_artifacts_file_store.rb
new file mode 100644
index 00000000000..1084ca14a34
--- /dev/null
+++ b/db/migrate/20180418053107_add_index_to_ci_job_artifacts_file_store.rb
@@ -0,0 +1,15 @@
+class AddIndexToCiJobArtifactsFileStore < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :ci_job_artifacts, :file_store
+  end
+
+  def down
+    remove_index :ci_job_artifacts, :file_store if index_exists?(:ci_job_artifacts, :file_store)
+  end
+end
diff --git a/db/migrate/20180425131009_assure_commits_count_for_merge_request_diff.rb b/db/migrate/20180425131009_assure_commits_count_for_merge_request_diff.rb
new file mode 100644
index 00000000000..0e991c23bfa
--- /dev/null
+++ b/db/migrate/20180425131009_assure_commits_count_for_merge_request_diff.rb
@@ -0,0 +1,27 @@
+class AssureCommitsCountForMergeRequestDiff < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class MergeRequestDiff < ActiveRecord::Base
+    self.table_name = 'merge_request_diffs'
+
+    include ::EachBatch
+  end
+
+  def up
+    Gitlab::BackgroundMigration.steal('AddMergeRequestDiffCommitsCount')
+
+    MergeRequestDiff.where(commits_count: nil).each_batch(of: 50) do |batch|
+      range = batch.pluck('MIN(id)', 'MAX(id)').first
+
+      Gitlab::BackgroundMigration::AddMergeRequestDiffCommitsCount.new.perform(*range)
+    end
+  end
+
+  def down
+    # noop
+  end
+end
diff --git a/db/migrate/20180430101916_add_runner_type_to_ci_runners.rb b/db/migrate/20180430101916_add_runner_type_to_ci_runners.rb
new file mode 100644
index 00000000000..42409349b75
--- /dev/null
+++ b/db/migrate/20180430101916_add_runner_type_to_ci_runners.rb
@@ -0,0 +1,9 @@
+class AddRunnerTypeToCiRunners < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_runners, :runner_type, :smallint
+  end
+end
diff --git a/db/migrate/20180503150427_add_index_to_namespaces_runners_token.rb b/db/migrate/20180503150427_add_index_to_namespaces_runners_token.rb
new file mode 100644
index 00000000000..4c4e576d49f
--- /dev/null
+++ b/db/migrate/20180503150427_add_index_to_namespaces_runners_token.rb
@@ -0,0 +1,20 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddIndexToNamespacesRunnersToken < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :namespaces, :runners_token, unique: true
+  end
+
+  def down
+    if index_exists?(:namespaces, :runners_token, unique: true)
+      remove_index :namespaces, :runners_token
+    end
+  end
+end
diff --git a/db/post_migrate/20180104131052_schedule_set_confidential_note_events_on_webhooks.rb b/db/post_migrate/20180104131052_schedule_set_confidential_note_events_on_webhooks.rb
new file mode 100644
index 00000000000..fa51ac83619
--- /dev/null
+++ b/db/post_migrate/20180104131052_schedule_set_confidential_note_events_on_webhooks.rb
@@ -0,0 +1,23 @@
+class ScheduleSetConfidentialNoteEventsOnWebhooks < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BATCH_SIZE = 1_000
+  INTERVAL = 5.minutes
+
+  disable_ddl_transaction!
+
+  def up
+    migration = Gitlab::BackgroundMigration::SetConfidentialNoteEventsOnWebhooks
+    migration_name = migration.to_s.demodulize
+    relation = migration::WebHook.hooks_to_update
+
+    queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                          migration_name,
+                                                          INTERVAL,
+                                                          batch_size: BATCH_SIZE)
+  end
+
+  def down
+  end
+end
diff --git a/db/post_migrate/20180122154930_schedule_set_confidential_note_events_on_services.rb b/db/post_migrate/20180122154930_schedule_set_confidential_note_events_on_services.rb
new file mode 100644
index 00000000000..a3ff9f1719e
--- /dev/null
+++ b/db/post_migrate/20180122154930_schedule_set_confidential_note_events_on_services.rb
@@ -0,0 +1,23 @@
+class ScheduleSetConfidentialNoteEventsOnServices < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BATCH_SIZE = 1_000
+  INTERVAL = 20.minutes
+
+  disable_ddl_transaction!
+
+  def up
+    migration = Gitlab::BackgroundMigration::SetConfidentialNoteEventsOnServices
+    migration_name = migration.to_s.demodulize
+    relation = migration::Service.services_to_update
+
+    queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                          migration_name,
+                                                          INTERVAL,
+                                                          batch_size: BATCH_SIZE)
+  end
+
+  def down
+  end
+end
diff --git a/db/post_migrate/20180212101928_schedule_build_stage_migration.rb b/db/post_migrate/20180212101928_schedule_build_stage_migration.rb
index df15b2cd9d4..0f61fa81832 100644
--- a/db/post_migrate/20180212101928_schedule_build_stage_migration.rb
+++ b/db/post_migrate/20180212101928_schedule_build_stage_migration.rb
@@ -1,26 +1,11 @@
 class ScheduleBuildStageMigration < ActiveRecord::Migration
-  include Gitlab::Database::MigrationHelpers
-
-  DOWNTIME = false
-  MIGRATION = 'MigrateBuildStage'.freeze
-  BATCH_SIZE = 500
-
-  disable_ddl_transaction!
-
-  class Build < ActiveRecord::Base
-    include EachBatch
-    self.table_name = 'ci_builds'
-  end
+  ##
+  # This migration has been rescheduled to run again, see
+  # `20180405101928_reschedule_builds_stages_migration.rb`
+  #
 
   def up
-    disable_statement_timeout
-
-    Build.where('stage_id IS NULL').tap do |relation|
-      queue_background_migration_jobs_by_range_at_intervals(relation,
-                                                            MIGRATION,
-                                                            5.minutes,
-                                                            batch_size: BATCH_SIZE)
-    end
+    # noop
   end
 
   def down
diff --git a/db/post_migrate/20180405101928_reschedule_builds_stages_migration.rb b/db/post_migrate/20180405101928_reschedule_builds_stages_migration.rb
new file mode 100644
index 00000000000..e19387bce1e
--- /dev/null
+++ b/db/post_migrate/20180405101928_reschedule_builds_stages_migration.rb
@@ -0,0 +1,33 @@
+class RescheduleBuildsStagesMigration < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  ##
+  # Rescheduled `20180212101928_schedule_build_stage_migration.rb`
+  #
+
+  DOWNTIME = false
+  MIGRATION = 'MigrateBuildStage'.freeze
+  BATCH_SIZE = 500
+
+  disable_ddl_transaction!
+
+  class Build < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'ci_builds'
+  end
+
+  def up
+    disable_statement_timeout
+
+    Build.where('stage_id IS NULL').tap do |relation|
+      queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                            MIGRATION,
+                                                            5.minutes,
+                                                            batch_size: BATCH_SIZE)
+    end
+  end
+
+  def down
+    # noop
+  end
+end
diff --git a/db/post_migrate/20180409170809_populate_missing_project_ci_cd_settings.rb b/db/post_migrate/20180409170809_populate_missing_project_ci_cd_settings.rb
new file mode 100644
index 00000000000..3b0fdb3aeea
--- /dev/null
+++ b/db/post_migrate/20180409170809_populate_missing_project_ci_cd_settings.rb
@@ -0,0 +1,34 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class PopulateMissingProjectCiCdSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    # MySQL does not support online upgrades, thus there can't be any missing
+    # rows.
+    return if Gitlab::Database.mysql?
+
+    # Projects created after the initial migration but before the code started
+    # using ProjectCiCdSetting won't have a corresponding row in
+    # project_ci_cd_settings, so let's fix that.
+    execute <<~SQL
+      INSERT INTO project_ci_cd_settings (project_id)
+      SELECT id
+      FROM projects
+      WHERE NOT EXISTS (
+        SELECT 1
+        FROM project_ci_cd_settings
+        WHERE project_ci_cd_settings.project_id = projects.id
+      )
+    SQL
+  end
+
+  def down
+    # There's nothing to revert for this migration.
+  end
+end
diff --git a/db/post_migrate/20180420080616_schedule_stages_index_migration.rb b/db/post_migrate/20180420080616_schedule_stages_index_migration.rb
new file mode 100644
index 00000000000..1d0daad002f
--- /dev/null
+++ b/db/post_migrate/20180420080616_schedule_stages_index_migration.rb
@@ -0,0 +1,29 @@
+class ScheduleStagesIndexMigration < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MIGRATION = 'MigrateStageIndex'.freeze
+  BATCH_SIZE = 10000
+
+  disable_ddl_transaction!
+
+  class Stage < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'ci_stages'
+  end
+
+  def up
+    disable_statement_timeout
+
+    Stage.all.tap do |relation|
+      queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                            MIGRATION,
+                                                            5.minutes,
+                                                            batch_size: BATCH_SIZE)
+    end
+  end
+
+  def down
+    # noop
+  end
+end
diff --git a/db/post_migrate/20180430143705_backfill_runner_type_for_ci_runners_post_migrate.rb b/db/post_migrate/20180430143705_backfill_runner_type_for_ci_runners_post_migrate.rb
new file mode 100644
index 00000000000..38af5aae924
--- /dev/null
+++ b/db/post_migrate/20180430143705_backfill_runner_type_for_ci_runners_post_migrate.rb
@@ -0,0 +1,23 @@
+class BackfillRunnerTypeForCiRunnersPostMigrate < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  INSTANCE_RUNNER_TYPE = 1
+  PROJECT_RUNNER_TYPE = 3
+
+  disable_ddl_transaction!
+
+  def up
+    update_column_in_batches(:ci_runners, :runner_type, INSTANCE_RUNNER_TYPE) do |table, query|
+      query.where(table[:is_shared].eq(true)).where(table[:runner_type].eq(nil))
+    end
+
+    update_column_in_batches(:ci_runners, :runner_type, PROJECT_RUNNER_TYPE) do |table, query|
+      query.where(table[:is_shared].eq(false)).where(table[:runner_type].eq(nil))
+    end
+  end
+
+  def down
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 06fc1a9d7e9..a37e6edc8d1 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180327101207) do
+ActiveRecord::Schema.define(version: 20180503150427) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -322,6 +322,7 @@ ActiveRecord::Schema.define(version: 20180327101207) do
   add_index "ci_builds", ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id", using: :btree
   add_index "ci_builds", ["protected"], name: "index_ci_builds_on_protected", using: :btree
   add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree
+  add_index "ci_builds", ["stage_id", "stage_idx"], name: "tmp_build_stage_position_index", where: "(stage_idx IS NOT NULL)", using: :btree
   add_index "ci_builds", ["stage_id"], name: "index_ci_builds_on_stage_id", using: :btree
   add_index "ci_builds", ["status", "type", "runner_id"], name: "index_ci_builds_on_status_and_type_and_runner_id", using: :btree
   add_index "ci_builds", ["status"], name: "index_ci_builds_on_status", using: :btree
@@ -367,6 +368,7 @@ ActiveRecord::Schema.define(version: 20180327101207) do
   end
 
   add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree
+  add_index "ci_job_artifacts", ["file_store"], name: "index_ci_job_artifacts_on_file_store", using: :btree
   add_index "ci_job_artifacts", ["job_id", "file_type"], name: "index_ci_job_artifacts_on_job_id_and_file_type", unique: true, using: :btree
   add_index "ci_job_artifacts", ["project_id"], name: "index_ci_job_artifacts_on_project_id", using: :btree
 
@@ -442,6 +444,14 @@ ActiveRecord::Schema.define(version: 20180327101207) do
   add_index "ci_pipelines", ["status"], name: "index_ci_pipelines_on_status", using: :btree
   add_index "ci_pipelines", ["user_id"], name: "index_ci_pipelines_on_user_id", using: :btree
 
+  create_table "ci_runner_namespaces", force: :cascade do |t|
+    t.integer "runner_id"
+    t.integer "namespace_id"
+  end
+
+  add_index "ci_runner_namespaces", ["namespace_id"], name: "index_ci_runner_namespaces_on_namespace_id", using: :btree
+  add_index "ci_runner_namespaces", ["runner_id", "namespace_id"], name: "index_ci_runner_namespaces_on_runner_id_and_namespace_id", unique: true, using: :btree
+
   create_table "ci_runner_projects", force: :cascade do |t|
     t.integer "runner_id", null: false
     t.datetime "created_at"
@@ -470,6 +480,7 @@ ActiveRecord::Schema.define(version: 20180327101207) do
     t.integer "access_level", default: 0, null: false
     t.string "ip_address"
     t.integer "maximum_timeout"
+    t.integer "runner_type", limit: 2
   end
 
   add_index "ci_runners", ["contacted_at"], name: "index_ci_runners_on_contacted_at", using: :btree
@@ -485,6 +496,7 @@ ActiveRecord::Schema.define(version: 20180327101207) do
     t.string "name"
     t.integer "status"
     t.integer "lock_version"
+    t.integer "position"
   end
 
   add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", unique: true, using: :btree
@@ -683,6 +695,19 @@ ActiveRecord::Schema.define(version: 20180327101207) do
 
   add_index "deploy_keys_projects", ["project_id"], name: "index_deploy_keys_projects_on_project_id", using: :btree
 
+  create_table "deploy_tokens", force: :cascade do |t|
+    t.boolean "revoked", default: false
+    t.boolean "read_repository", default: false, null: false
+    t.boolean "read_registry", default: false, null: false
+    t.datetime_with_timezone "expires_at", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.string "name", null: false
+    t.string "token", null: false
+  end
+
+  add_index "deploy_tokens", ["token", "expires_at", "id"], name: "index_deploy_tokens_on_token_and_expires_at_and_id", where: "(revoked IS FALSE)", using: :btree
+  add_index "deploy_tokens", ["token"], name: "index_deploy_tokens_on_token", unique: true, using: :btree
+
   create_table "deployments", force: :cascade do |t|
     t.integer "iid", null: false
     t.integer "project_id", null: false
@@ -882,12 +907,14 @@ ActiveRecord::Schema.define(version: 20180327101207) do
   add_index "identities", ["user_id"], name: "index_identities_on_user_id", using: :btree
 
   create_table "internal_ids", id: :bigserial, force: :cascade do |t|
-    t.integer "project_id", null: false
+    t.integer "project_id"
     t.integer "usage", null: false
     t.integer "last_value", null: false
+    t.integer "namespace_id"
   end
 
-  add_index "internal_ids", ["usage", "project_id"], name: "index_internal_ids_on_usage_and_project_id", unique: true, using: :btree
+  add_index "internal_ids", ["usage", "namespace_id"], name: "index_internal_ids_on_usage_and_namespace_id", unique: true, where: "(namespace_id IS NOT NULL)", using: :btree
+  add_index "internal_ids", ["usage", "project_id"], name: "index_internal_ids_on_usage_and_project_id", unique: true, where: "(project_id IS NOT NULL)", using: :btree
 
   create_table "issue_assignees", id: false, force: :cascade do |t|
     t.integer "user_id", null: false
@@ -1243,6 +1270,7 @@ ActiveRecord::Schema.define(version: 20180327101207) do
     t.boolean "require_two_factor_authentication", default: false, null: false
     t.integer "two_factor_grace_period", default: 48, null: false
     t.integer "cached_markdown_version"
+    t.string "runners_token"
   end
 
   add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree
@@ -1253,6 +1281,7 @@ ActiveRecord::Schema.define(version: 20180327101207) do
   add_index "namespaces", ["path"], name: "index_namespaces_on_path", using: :btree
   add_index "namespaces", ["path"], name: "index_namespaces_on_path_trigram", using: :gin, opclasses: {"path"=>"gin_trgm_ops"}
   add_index "namespaces", ["require_two_factor_authentication"], name: "index_namespaces_on_require_two_factor_authentication", using: :btree
+  add_index "namespaces", ["runners_token"], name: "index_namespaces_on_runners_token", unique: true, using: :btree
   add_index "namespaces", ["type"], name: "index_namespaces_on_type", using: :btree
 
   create_table "notes", force: :cascade do |t|
@@ -1312,6 +1341,7 @@ ActiveRecord::Schema.define(version: 20180327101207) do
     t.boolean "failed_pipeline"
     t.boolean "success_pipeline"
     t.boolean "push_to_merge_request"
+    t.boolean "issue_due"
   end
 
   add_index "notification_settings", ["source_id", "source_type"], name: "index_notification_settings_on_source_id_and_source_type", using: :btree
@@ -1419,6 +1449,13 @@ ActiveRecord::Schema.define(version: 20180327101207) do
 
   add_index "project_auto_devops", ["project_id"], name: "index_project_auto_devops_on_project_id", unique: true, using: :btree
 
+  create_table "project_ci_cd_settings", force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.boolean "group_runners_enabled", default: true, null: false
+  end
+
+  add_index "project_ci_cd_settings", ["project_id"], name: "index_project_ci_cd_settings_on_project_id", unique: true, using: :btree
+
   create_table "project_custom_attributes", force: :cascade do |t|
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
@@ -1430,6 +1467,14 @@ ActiveRecord::Schema.define(version: 20180327101207) do
   add_index "project_custom_attributes", ["key", "value"], name: "index_project_custom_attributes_on_key_and_value", using: :btree
   add_index "project_custom_attributes", ["project_id", "key"], name: "index_project_custom_attributes_on_project_id_and_key", unique: true, using: :btree
 
+  create_table "project_deploy_tokens", force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.integer "deploy_token_id", null: false
+    t.datetime_with_timezone "created_at", null: false
+  end
+
+  add_index "project_deploy_tokens", ["project_id", "deploy_token_id"], name: "index_project_deploy_tokens_on_project_id_and_deploy_token_id", unique: true, using: :btree
+
   create_table "project_features", force: :cascade do |t|
     t.integer "project_id"
     t.integer "merge_requests_access_level"
@@ -1684,6 +1729,7 @@ ActiveRecord::Schema.define(version: 20180327101207) do
     t.boolean "confidential_issues_events", default: true, null: false
     t.boolean "commit_events", default: true, null: false
     t.boolean "job_events", default: false, null: false
+    t.boolean "confidential_note_events", default: true
   end
 
   add_index "services", ["project_id"], name: "index_services_on_project_id", using: :btree
@@ -2022,6 +2068,7 @@ ActiveRecord::Schema.define(version: 20180327101207) do
     t.boolean "confidential_issues_events", default: false, null: false
     t.boolean "repository_update_events", default: false, null: false
     t.boolean "job_events", default: false, null: false
+    t.boolean "confidential_note_events"
   end
 
   add_index "web_hooks", ["project_id"], name: "index_web_hooks_on_project_id", using: :btree
@@ -2051,6 +2098,8 @@ ActiveRecord::Schema.define(version: 20180327101207) do
   add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify
   add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify
   add_foreign_key "ci_pipelines", "projects", name: "fk_86635dbd80", on_delete: :cascade
+  add_foreign_key "ci_runner_namespaces", "ci_runners", column: "runner_id", on_delete: :cascade
+  add_foreign_key "ci_runner_namespaces", "namespaces", on_delete: :cascade
   add_foreign_key "ci_runner_projects", "projects", name: "fk_4478a6f1e4", on_delete: :cascade
   add_foreign_key "ci_stages", "ci_pipelines", column: "pipeline_id", name: "fk_fb57e6cc56", on_delete: :cascade
   add_foreign_key "ci_stages", "projects", name: "fk_2360681d1d", on_delete: :cascade
@@ -2088,6 +2137,7 @@ ActiveRecord::Schema.define(version: 20180327101207) do
   add_foreign_key "gpg_signatures", "gpg_keys", on_delete: :nullify
   add_foreign_key "gpg_signatures", "projects", on_delete: :cascade
   add_foreign_key "group_custom_attributes", "namespaces", column: "group_id", on_delete: :cascade
+  add_foreign_key "internal_ids", "namespaces", name: "fk_162941d509", on_delete: :cascade
   add_foreign_key "internal_ids", "projects", on_delete: :cascade
   add_foreign_key "issue_assignees", "issues", name: "fk_b7d881734a", on_delete: :cascade
   add_foreign_key "issue_assignees", "users", name: "fk_5e0c8d9154", on_delete: :cascade
@@ -2134,7 +2184,10 @@ ActiveRecord::Schema.define(version: 20180327101207) do
   add_foreign_key "project_authorizations", "projects", on_delete: :cascade
   add_foreign_key "project_authorizations", "users", on_delete: :cascade
   add_foreign_key "project_auto_devops", "projects", on_delete: :cascade
+  add_foreign_key "project_ci_cd_settings", "projects", name: "fk_24c15d2f2e", on_delete: :cascade
   add_foreign_key "project_custom_attributes", "projects", on_delete: :cascade
+  add_foreign_key "project_deploy_tokens", "deploy_tokens", on_delete: :cascade
+  add_foreign_key "project_deploy_tokens", "projects", on_delete: :cascade
   add_foreign_key "project_features", "projects", name: "fk_18513d9b92", on_delete: :cascade
   add_foreign_key "project_group_links", "projects", name: "fk_daa8cee94c", on_delete: :cascade
   add_foreign_key "project_import_data", "projects", name: "fk_ffb9ee3a10", on_delete: :cascade