Merge branch 'master' into backstage/gb/migrate-stages-statuses

* master: (587 commits)
  Increase GitLab QA screenshots size
  Fix QA test scenario for creating a new project
  Configure headless chrome screenshots correctly
  Allow passing exceptions when creating project services
  Also inject new route helpers into includers of GitlabRoutingHelper
  Remove many N+1 queries with merge requests API
  fix milestones finder failing spec
  Make wrong migrations idempotent
  Remove double border on last group row
  Update tests to test what the component renders and what the clicks calls
  Username and password are no longer stripped from import url on import
  Change `sign_out` usage to `gitlab_sign_out` in one spec
  Send Gitaly Repository with /api/internal/allowed
  Resolve "Liberation Mono weird font rendering on Fedora/openSUSE/other Linux distros"
  Force mobile view for admin projects
  fix spec
  Mark a subgroup-related spec as PostgreSQL-only
  Use vue .prevent to prevent the click
  Add members-list class to admin projects page
  Fix another typo in SHA attribute spec
  ...

Conflicts:
	db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb
	db/schema.rb
	lib/gitlab/background_migration/migrate_build_stage_id_reference.rb
	spec/migrations/migrate_stage_id_reference_in_background_spec.rb

21 files changed, 675 insertions, 24 deletions

diff --git a/db/migrate/20160804142904_add_ci_config_file_to_project.rb b/db/migrate/20160804142904_add_ci_config_file_to_project.rb
new file mode 100644
index 00000000000..341ae555c1b
--- /dev/null
+++ b/db/migrate/20160804142904_add_ci_config_file_to_project.rb
@@ -0,0 +1,11 @@
+class AddCiConfigFileToProject < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :projects, :ci_config_path, :string
+  end
+
+  def down
+    remove_column :projects, :ci_config_path
+  end
+end
diff --git a/db/migrate/20170525130346_create_group_variables_table.rb b/db/migrate/20170525130346_create_group_variables_table.rb
new file mode 100644
index 00000000000..eaa38dbc40d
--- /dev/null
+++ b/db/migrate/20170525130346_create_group_variables_table.rb
@@ -0,0 +1,23 @@
+class CreateGroupVariablesTable < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    create_table :ci_group_variables do |t|
+      t.string :key, null: false
+      t.text :value
+      t.text :encrypted_value
+      t.string :encrypted_value_salt
+      t.string :encrypted_value_iv
+      t.integer :group_id, null: false
+      t.boolean :protected, default: false, null: false
+
+      t.timestamps_with_timezone null: false
+    end
+
+    add_index :ci_group_variables, [:group_id, :key], unique: true
+  end
+
+  def down
+    drop_table :ci_group_variables
+  end
+end
diff --git a/db/migrate/20170525130758_add_foreign_key_to_group_variables.rb b/db/migrate/20170525130758_add_foreign_key_to_group_variables.rb
new file mode 100644
index 00000000000..0146235c5ba
--- /dev/null
+++ b/db/migrate/20170525130758_add_foreign_key_to_group_variables.rb
@@ -0,0 +1,15 @@
+class AddForeignKeyToGroupVariables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :ci_group_variables, :namespaces, column: :group_id
+  end
+
+  def down
+    remove_foreign_key :ci_group_variables, column: :group_id
+  end
+end
diff --git a/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb b/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb
new file mode 100644
index 00000000000..3eaafac321d
--- /dev/null
+++ b/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb
@@ -0,0 +1,187 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  CONCURRENCY = 4
+
+  disable_ddl_transaction!
+
+  # The tables/columns for which to remove orphans and add foreign keys. Order
+  # matters as some tables/columns should be processed before others.
+  TABLES = [
+    [:boards, :projects, :project_id],
+    [:lists, :labels, :label_id],
+    [:lists, :boards, :board_id],
+    [:services, :projects, :project_id],
+    [:forked_project_links, :projects, :forked_to_project_id],
+    [:merge_requests, :projects, :target_project_id],
+    [:labels, :projects, :project_id],
+    [:issues, :projects, :project_id],
+    [:events, :projects, :project_id],
+    [:milestones, :projects, :project_id],
+    [:notes, :projects, :project_id],
+    [:snippets, :projects, :project_id],
+    [:web_hooks, :projects, :project_id],
+    [:protected_branch_merge_access_levels, :protected_branches, :protected_branch_id],
+    [:protected_branch_push_access_levels, :protected_branches, :protected_branch_id],
+    [:protected_branches, :projects, :project_id],
+    [:protected_tags, :projects, :project_id],
+    [:deploy_keys_projects, :projects, :project_id],
+    [:users_star_projects, :projects, :project_id],
+    [:releases, :projects, :project_id],
+    [:project_group_links, :projects, :project_id],
+    [:pages_domains, :projects, :project_id],
+    [:todos, :projects, :project_id],
+    [:project_import_data, :projects, :project_id],
+    [:project_features, :projects, :project_id],
+    [:ci_builds, :projects, :project_id],
+    [:ci_pipelines, :projects, :project_id],
+    [:ci_runner_projects, :projects, :project_id],
+    [:ci_triggers, :projects, :project_id],
+    [:environments, :projects, :project_id],
+    [:deployments, :projects, :project_id]
+  ]
+
+  def up
+    # These existing foreign keys don't have an "ON DELETE CASCADE" clause.
+    remove_foreign_key_without_error(:boards, :project_id)
+    remove_foreign_key_without_error(:lists, :label_id)
+    remove_foreign_key_without_error(:lists, :board_id)
+    remove_foreign_key_without_error(:protected_branch_merge_access_levels,
+                                     :protected_branch_id)
+
+    remove_foreign_key_without_error(:protected_branch_push_access_levels,
+                                     :protected_branch_id)
+
+    remove_orphaned_rows
+    add_foreign_keys
+
+    # These columns are not indexed yet, meaning a cascading delete would take
+    # forever.
+    add_concurrent_index(:project_group_links, :project_id)
+    add_concurrent_index(:pages_domains, :project_id)
+  end
+
+  def down
+    TABLES.each do |(source, _, column)|
+      remove_foreign_key_without_error(source, column)
+    end
+
+    add_concurrent_foreign_key(:boards, :projects, column: :project_id)
+    add_concurrent_foreign_key(:lists, :labels, column: :label_id)
+    add_concurrent_foreign_key(:lists, :boards, column: :board_id)
+
+    add_concurrent_foreign_key(:protected_branch_merge_access_levels,
+                               :protected_branches,
+                               column: :protected_branch_id)
+
+    add_concurrent_foreign_key(:protected_branch_push_access_levels,
+                               :protected_branches,
+                               column: :protected_branch_id)
+
+    remove_index_without_error(:project_group_links, :project_id)
+    remove_index_without_error(:pages_domains, :project_id)
+  end
+
+  def add_foreign_keys
+    TABLES.each do |(source, target, column)|
+      add_concurrent_foreign_key(source, target, column: column)
+    end
+  end
+
+  # Removes orphans from various tables concurrently.
+  def remove_orphaned_rows
+    Gitlab::Database.with_connection_pool(CONCURRENCY) do |pool|
+      queues = queues_for_rows(TABLES)
+
+      threads = queues.map do |queue|
+        Thread.new do
+          pool.with_connection do |connection|
+            Thread.current[:foreign_key_connection] = connection
+
+            # Disables statement timeouts for the current connection. This is
+            # necessary as removing of orphaned data might otherwise exceed the
+            # statement timeout.
+            disable_statement_timeout
+
+            remove_orphans(*queue.pop) until queue.empty?
+
+            steal_from_queues(queues - [queue])
+          end
+        end
+      end
+
+      threads.each(&:join)
+    end
+  end
+
+  def steal_from_queues(queues)
+    loop do
+      stolen = false
+
+      queues.each do |queue|
+        # Stealing is racy so it's possible a pop might be called on an
+        # already-empty queue.
+        begin
+          remove_orphans(*queue.pop(true))
+          stolen = true
+        rescue ThreadError
+        end
+      end
+
+      break unless stolen
+    end
+  end
+
+  def remove_orphans(source, target, column)
+    quoted_source = quote_table_name(source)
+    quoted_target = quote_table_name(target)
+    quoted_column = quote_column_name(column)
+
+    execute <<-EOF.strip_heredoc
+    DELETE FROM #{quoted_source}
+    WHERE NOT EXISTS (
+      SELECT true
+      FROM #{quoted_target}
+      WHERE #{quoted_target}.id = #{quoted_source}.#{quoted_column}
+    )
+    AND #{quoted_source}.#{quoted_column} IS NOT NULL
+    EOF
+  end
+
+  def remove_foreign_key_without_error(table, column)
+    remove_foreign_key(table, column: column)
+  rescue ArgumentError
+  end
+
+  def remove_index_without_error(table, column)
+    remove_concurrent_index(table, column)
+  rescue ArgumentError
+  end
+
+  def connection
+    # Rails memoizes connection objects, but this causes them to be shared
+    # amongst threads; we don't want that.
+    Thread.current[:foreign_key_connection] || ActiveRecord::Base.connection
+  end
+
+  def queues_for_rows(rows)
+    queues = Array.new(CONCURRENCY) { Queue.new }
+    slice_size = rows.length / CONCURRENCY
+
+    # Divide all the tuples as evenly as possible amongst the queues.
+    rows.each_slice(slice_size).each_with_index do |slice, index|
+      bucket = index % CONCURRENCY
+
+      slice.each do |row|
+        queues[bucket] << row
+      end
+    end
+
+    queues
+  end
+end
diff --git a/db/migrate/20170616133147_create_merge_request_diff_commits.rb b/db/migrate/20170616133147_create_merge_request_diff_commits.rb
new file mode 100644
index 00000000000..616464cb470
--- /dev/null
+++ b/db/migrate/20170616133147_create_merge_request_diff_commits.rb
@@ -0,0 +1,20 @@
+class CreateMergeRequestDiffCommits < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    create_table :merge_request_diff_commits, id: false do |t|
+      t.datetime_with_timezone :authored_date
+      t.datetime_with_timezone :committed_date
+      t.belongs_to :merge_request_diff, null: false, foreign_key: { on_delete: :cascade }
+      t.integer :relative_order, null: false
+      t.binary :sha, null: false, limit: 20
+      t.text :author_name
+      t.text :author_email
+      t.text :committer_name
+      t.text :committer_email
+      t.text :message
+
+      t.index [:merge_request_diff_id, :relative_order], name: 'index_merge_request_diff_commits_on_mr_diff_id_and_order', unique: true
+    end
+  end
+end
diff --git a/db/migrate/20170620064728_create_ci_pipeline_schedule_variables.rb b/db/migrate/20170620064728_create_ci_pipeline_schedule_variables.rb
new file mode 100644
index 00000000000..92833765a82
--- /dev/null
+++ b/db/migrate/20170620064728_create_ci_pipeline_schedule_variables.rb
@@ -0,0 +1,25 @@
+class CreateCiPipelineScheduleVariables < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    create_table :ci_pipeline_schedule_variables do |t|
+      t.string :key, null: false
+      t.text :value
+      t.text :encrypted_value
+      t.string :encrypted_value_salt
+      t.string :encrypted_value_iv
+      t.integer :pipeline_schedule_id, null: false
+
+      t.timestamps_with_timezone null: true
+    end
+
+    add_index :ci_pipeline_schedule_variables,
+      [:pipeline_schedule_id, :key],
+      name: "index_ci_pipeline_schedule_variables_on_schedule_id_and_key",
+      unique: true
+  end
+
+  def down
+    drop_table :ci_pipeline_schedule_variables
+  end
+end
diff --git a/db/migrate/20170620065449_add_foreign_key_to_ci_pipeline_schedule_variables.rb b/db/migrate/20170620065449_add_foreign_key_to_ci_pipeline_schedule_variables.rb
new file mode 100644
index 00000000000..7bbf66e0ac3
--- /dev/null
+++ b/db/migrate/20170620065449_add_foreign_key_to_ci_pipeline_schedule_variables.rb
@@ -0,0 +1,15 @@
+class AddForeignKeyToCiPipelineScheduleVariables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key(:ci_pipeline_schedule_variables, :ci_pipeline_schedules, column: :pipeline_schedule_id)
+  end
+
+  def down
+    remove_foreign_key(:ci_pipeline_schedule_variables, column: :pipeline_schedule_id)
+  end
+end
diff --git a/db/migrate/20170622130029_correct_protected_branches_foreign_keys.rb b/db/migrate/20170622130029_correct_protected_branches_foreign_keys.rb
new file mode 100644
index 00000000000..46497775527
--- /dev/null
+++ b/db/migrate/20170622130029_correct_protected_branches_foreign_keys.rb
@@ -0,0 +1,40 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CorrectProtectedBranchesForeignKeys < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    remove_foreign_key_without_error(:protected_branch_push_access_levels,
+                                     column: :protected_branch_id)
+
+    execute <<-EOF
+    DELETE FROM protected_branch_push_access_levels
+    WHERE NOT EXISTS (
+      SELECT true
+      FROM protected_branches
+      WHERE protected_branch_push_access_levels.protected_branch_id = protected_branches.id
+    )
+    AND protected_branch_id IS NOT NULL
+    EOF
+
+    add_concurrent_foreign_key(:protected_branch_push_access_levels,
+                               :protected_branches,
+                               column: :protected_branch_id)
+  end
+
+  def down
+    # Previously there was a foreign key without a CASCADING DELETE, so we'll
+    # just leave the foreign key in place.
+  end
+
+  def remove_foreign_key_without_error(*args)
+    remove_foreign_key(*args)
+  rescue ArgumentError
+  end
+end
diff --git a/db/migrate/20170622132212_add_foreign_key_for_merge_request_diffs.rb b/db/migrate/20170622132212_add_foreign_key_for_merge_request_diffs.rb
new file mode 100644
index 00000000000..9f524fac8a7
--- /dev/null
+++ b/db/migrate/20170622132212_add_foreign_key_for_merge_request_diffs.rb
@@ -0,0 +1,30 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddForeignKeyForMergeRequestDiffs < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    execute <<-EOF
+    DELETE FROM merge_request_diffs
+    WHERE NOT EXISTS (
+      SELECT true
+      FROM merge_requests
+      WHERE merge_requests.id = merge_request_diffs.merge_request_id
+    )
+    EOF
+
+    add_concurrent_foreign_key(:merge_request_diffs,
+                               :merge_requests,
+                               column: :merge_request_id)
+  end
+
+  def down
+    remove_foreign_key(:merge_request_diffs, column: :merge_request_id)
+  end
+end
diff --git a/db/migrate/20170622135451_rename_duplicated_variable_key.rb b/db/migrate/20170622135451_rename_duplicated_variable_key.rb
new file mode 100644
index 00000000000..368718ab0ce
--- /dev/null
+++ b/db/migrate/20170622135451_rename_duplicated_variable_key.rb
@@ -0,0 +1,38 @@
+class RenameDuplicatedVariableKey < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    execute(<<~SQL)
+      UPDATE ci_variables
+      SET #{key} = CONCAT(#{key}, #{underscore}, id)
+      WHERE id IN (
+        SELECT *
+        FROM ( -- MySQL requires an extra layer
+          SELECT dup.id
+          FROM ci_variables dup
+          INNER JOIN (SELECT max(id) AS id, #{key}, project_id
+                      FROM ci_variables tmp
+                      GROUP BY #{key}, project_id) var
+          USING (#{key}, project_id) where dup.id <> var.id
+        ) dummy
+      )
+    SQL
+  end
+
+  def down
+    # noop
+  end
+
+  def key
+    # key needs to be quoted in MySQL
+    quote_column_name('key')
+  end
+
+  def underscore
+    quote('_')
+  end
+end
diff --git a/db/migrate/20170622135628_add_environment_scope_to_ci_variables.rb b/db/migrate/20170622135628_add_environment_scope_to_ci_variables.rb
new file mode 100644
index 00000000000..17fe062d8d5
--- /dev/null
+++ b/db/migrate/20170622135628_add_environment_scope_to_ci_variables.rb
@@ -0,0 +1,15 @@
+class AddEnvironmentScopeToCiVariables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:ci_variables, :environment_scope, :string, default: '*')
+  end
+
+  def down
+    remove_column(:ci_variables, :environment_scope)
+  end
+end
diff --git a/db/migrate/20170622135728_add_unique_constraint_to_ci_variables.rb b/db/migrate/20170622135728_add_unique_constraint_to_ci_variables.rb
new file mode 100644
index 00000000000..8b2cc40ee59
--- /dev/null
+++ b/db/migrate/20170622135728_add_unique_constraint_to_ci_variables.rb
@@ -0,0 +1,38 @@
+class AddUniqueConstraintToCiVariables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    unless this_index_exists?
+      add_concurrent_index(:ci_variables, columns, name: index_name, unique: true)
+    end
+  end
+
+  def down
+    if this_index_exists?
+      if Gitlab::Database.mysql? && !index_exists?(:ci_variables, :project_id)
+        # Need to add this index for MySQL project_id foreign key constraint
+        add_concurrent_index(:ci_variables, :project_id)
+      end
+
+      remove_concurrent_index(:ci_variables, columns, name: index_name)
+    end
+  end
+
+  private
+
+  def this_index_exists?
+    index_exists?(:ci_variables, columns, name: index_name)
+  end
+
+  def columns
+    @columns ||= [:project_id, :key, :environment_scope]
+  end
+
+  def index_name
+    'index_ci_variables_on_project_id_and_key_and_environment_scope'
+  end
+end
diff --git a/db/migrate/20170623080805_remove_ci_variables_project_id_index.rb b/db/migrate/20170623080805_remove_ci_variables_project_id_index.rb
new file mode 100644
index 00000000000..ddcc0292b9d
--- /dev/null
+++ b/db/migrate/20170623080805_remove_ci_variables_project_id_index.rb
@@ -0,0 +1,19 @@
+class RemoveCiVariablesProjectIdIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    if index_exists?(:ci_variables, :project_id)
+      remove_concurrent_index(:ci_variables, :project_id)
+    end
+  end
+
+  def down
+    unless index_exists?(:ci_variables, :project_id)
+      add_concurrent_index(:ci_variables, :project_id)
+    end
+  end
+end
diff --git a/db/migrate/20170703102400_add_stage_id_foreign_key_to_builds.rb b/db/migrate/20170703102400_add_stage_id_foreign_key_to_builds.rb
new file mode 100644
index 00000000000..68b947583d3
--- /dev/null
+++ b/db/migrate/20170703102400_add_stage_id_foreign_key_to_builds.rb
@@ -0,0 +1,35 @@
+class AddStageIdForeignKeyToBuilds < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    unless index_exists?(:ci_builds, :stage_id)
+      add_concurrent_index(:ci_builds, :stage_id)
+    end
+
+    unless foreign_key_exists?(:ci_builds, :stage_id)
+      add_concurrent_foreign_key(:ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade)
+    end
+  end
+
+  def down
+    if foreign_key_exists?(:ci_builds, :stage_id)
+      remove_foreign_key(:ci_builds, column: :stage_id)
+    end
+
+    if index_exists?(:ci_builds, :stage_id)
+      remove_concurrent_index(:ci_builds, :stage_id)
+    end
+  end
+
+  private
+
+  def foreign_key_exists?(table, column)
+    foreign_keys(:ci_builds).any? do |key|
+      key.options[:column] == column.to_s
+    end
+  end
+end
diff --git a/db/migrate/20170706151212_add_performance_bar_allowed_group_id_to_application_settings.rb b/db/migrate/20170706151212_add_performance_bar_allowed_group_id_to_application_settings.rb
new file mode 100644
index 00000000000..fe9970ddc71
--- /dev/null
+++ b/db/migrate/20170706151212_add_performance_bar_allowed_group_id_to_application_settings.rb
@@ -0,0 +1,9 @@
+class AddPerformanceBarAllowedGroupIdToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :performance_bar_allowed_group_id, :integer
+  end
+end
diff --git a/db/migrate/20170707183807_add_group_id_to_milestones.rb b/db/migrate/20170707183807_add_group_id_to_milestones.rb
new file mode 100644
index 00000000000..675ffd4a1c9
--- /dev/null
+++ b/db/migrate/20170707183807_add_group_id_to_milestones.rb
@@ -0,0 +1,20 @@
+class AddGroupIdToMilestones < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    return if column_exists? :milestones, :group_id
+
+    change_column_null :milestones, :project_id, true
+
+    add_column :milestones, :group_id, :integer
+  end
+
+  def down
+    # We cannot rollback project_id not null constraint if there are records
+    # with null values.
+    execute "DELETE from milestones WHERE project_id IS NULL"
+
+    remove_column :milestones, :group_id
+    change_column :milestones, :project_id, :integer, null: false
+  end
+end
diff --git a/db/migrate/20170707184243_add_group_milestone_id_indexes.rb b/db/migrate/20170707184243_add_group_milestone_id_indexes.rb
new file mode 100644
index 00000000000..aa48fe90cad
--- /dev/null
+++ b/db/migrate/20170707184243_add_group_milestone_id_indexes.rb
@@ -0,0 +1,21 @@
+class AddGroupMilestoneIdIndexes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    return if index_exists?(:milestones, :group_id)
+
+    add_concurrent_foreign_key :milestones, :namespaces, column: :group_id, on_delete: :cascade
+
+    add_concurrent_index :milestones, :group_id
+  end
+
+  def down
+    remove_foreign_key :milestones, column: :group_id
+
+    remove_concurrent_index :milestones, :group_id
+  end
+end
diff --git a/db/migrate/20170707184244_remove_wrong_versions_from_schema_versions.rb b/db/migrate/20170707184244_remove_wrong_versions_from_schema_versions.rb
new file mode 100644
index 00000000000..38536a8b06a
--- /dev/null
+++ b/db/migrate/20170707184244_remove_wrong_versions_from_schema_versions.rb
@@ -0,0 +1,10 @@
+class RemoveWrongVersionsFromSchemaVersions < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    execute("DELETE FROM schema_migrations WHERE version IN ('20170723183807', '20170724184243')")
+  end
+
+  def down
+  end
+end
diff --git a/db/post_migrate/20170621102400_add_stage_id_index_to_builds.rb b/db/post_migrate/20170621102400_add_stage_id_index_to_builds.rb
index 7d6609b18bf..ac61b5c84a8 100644
--- a/db/post_migrate/20170621102400_add_stage_id_index_to_builds.rb
+++ b/db/post_migrate/20170621102400_add_stage_id_index_to_builds.rb
@@ -3,19 +3,15 @@ class AddStageIdIndexToBuilds < ActiveRecord::Migration
 
   DOWNTIME = false
 
-  disable_ddl_transaction!
+  ##
+  # Improved in 20170703102400_add_stage_id_foreign_key_to_builds.rb
+  #
 
   def up
-    unless index_exists?(:ci_builds, :stage_id)
-      add_concurrent_foreign_key(:ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade)
-      add_concurrent_index(:ci_builds, :stage_id)
-    end
+    # noop
   end
 
   def down
-    if index_exists?(:ci_builds, :stage_id)
-      remove_foreign_key(:ci_builds, column: :stage_id)
-      remove_concurrent_index(:ci_builds, :stage_id)
-    end
+    # noop
   end
 end
diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb
index ebec4cb6bb7..f31015d77a3 100644
--- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb
+++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb
@@ -3,23 +3,27 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration
 
   DOWNTIME = false
   BATCH_SIZE = 10000
+  RANGE_SIZE = 1000
   MIGRATION = 'MigrateBuildStageIdReference'.freeze
 
   disable_ddl_transaction!
 
   class Build < ActiveRecord::Base
     self.table_name = 'ci_builds'
+    include ::EachBatch
   end
 
+  ##
+  # It will take around 3 days to process 20M ci_builds.
+  #
   def up
-    index = 1
+    Build.where(stage_id: nil).each_batch(of: BATCH_SIZE) do |relation, index|
+      relation.each_batch(of: RANGE_SIZE) do |relation|
+        range = relation.pluck('MIN(id)', 'MAX(id)').first
 
-    Build.where(stage_id: nil).in_batches(of: BATCH_SIZE) do |relation|
-      jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] }
-      schedule = index * 2.minutes
-      index += 1
-
-      BackgroundMigrationWorker.perform_bulk_in(schedule, jobs)
+        BackgroundMigrationWorker
+          .perform_in(index * 2.minutes, MIGRATION, range)
+      end
     end
   end
 
diff --git a/db/schema.rb b/db/schema.rb
index f34dd32fb74..e0c9e5efb33 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170630111158) do
+ActiveRecord::Schema.define(version: 20170707184244) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -126,6 +126,7 @@ ActiveRecord::Schema.define(version: 20170630111158) do
     t.boolean "prometheus_metrics_enabled", default: false, null: false
     t.boolean "help_page_hide_commercial_content", default: false
     t.string "help_page_support_url"
+    t.integer "performance_bar_allowed_group_id"
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -253,6 +254,33 @@ ActiveRecord::Schema.define(version: 20170630111158) do
   add_index "ci_builds", ["updated_at"], name: "index_ci_builds_on_updated_at", using: :btree
   add_index "ci_builds", ["user_id"], name: "index_ci_builds_on_user_id", using: :btree
 
+  create_table "ci_pipeline_schedule_variables", force: :cascade do |t|
+    t.string "key", null: false
+    t.text "value"
+    t.text "encrypted_value"
+    t.string "encrypted_value_salt"
+    t.string "encrypted_value_iv"
+    t.integer "pipeline_schedule_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "ci_pipeline_schedule_variables", ["pipeline_schedule_id", "key"], name: "index_ci_pipeline_schedule_variables_on_schedule_id_and_key", unique: true, using: :btree
+
+  create_table "ci_group_variables", force: :cascade do |t|
+    t.string "key", null: false
+    t.text "value"
+    t.text "encrypted_value"
+    t.string "encrypted_value_salt"
+    t.string "encrypted_value_iv"
+    t.integer "group_id", null: false
+    t.boolean "protected", default: false, null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "ci_group_variables", ["group_id", "key"], name: "index_ci_group_variables_on_group_id_and_key", unique: true, using: :btree
+
   create_table "ci_pipeline_schedules", force: :cascade do |t|
     t.string "description"
     t.string "ref"
@@ -375,9 +403,10 @@ ActiveRecord::Schema.define(version: 20170630111158) do
     t.string "encrypted_value_iv"
     t.integer "project_id", null: false
     t.boolean "protected", default: false, null: false
+    t.string "environment_scope", default: "*", null: false
   end
 
-  add_index "ci_variables", ["project_id"], name: "index_ci_variables_on_project_id", using: :btree
+  add_index "ci_variables", ["project_id", "key", "environment_scope"], name: "index_ci_variables_on_project_id_and_key_and_environment_scope", unique: true, using: :btree
 
   create_table "container_repositories", force: :cascade do |t|
     t.integer "project_id", null: false
@@ -693,6 +722,21 @@ ActiveRecord::Schema.define(version: 20170630111158) do
   add_index "members", ["source_id", "source_type"], name: "index_members_on_source_id_and_source_type", using: :btree
   add_index "members", ["user_id"], name: "index_members_on_user_id", using: :btree
 
+  create_table "merge_request_diff_commits", id: false, force: :cascade do |t|
+    t.datetime "authored_date"
+    t.datetime "committed_date"
+    t.integer "merge_request_diff_id", null: false
+    t.integer "relative_order", null: false
+    t.binary "sha", null: false
+    t.text "author_name"
+    t.text "author_email"
+    t.text "committer_name"
+    t.text "committer_email"
+    t.text "message"
+  end
+
+  add_index "merge_request_diff_commits", ["merge_request_diff_id", "relative_order"], name: "index_merge_request_diff_commits_on_mr_diff_id_and_order", unique: true, using: :btree
+
   create_table "merge_request_diff_files", id: false, force: :cascade do |t|
     t.integer "merge_request_diff_id", null: false
     t.integer "relative_order", null: false
@@ -800,7 +844,7 @@ ActiveRecord::Schema.define(version: 20170630111158) do
 
   create_table "milestones", force: :cascade do |t|
     t.string "title", null: false
-    t.integer "project_id", null: false
+    t.integer "project_id"
     t.text "description"
     t.date "due_date"
     t.datetime "created_at"
@@ -811,10 +855,12 @@ ActiveRecord::Schema.define(version: 20170630111158) do
     t.text "description_html"
     t.date "start_date"
     t.integer "cached_markdown_version"
+    t.integer "group_id"
   end
 
   add_index "milestones", ["description"], name: "index_milestones_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
   add_index "milestones", ["due_date"], name: "index_milestones_on_due_date", using: :btree
+  add_index "milestones", ["group_id"], name: "index_milestones_on_group_id", using: :btree
   add_index "milestones", ["project_id", "iid"], name: "index_milestones_on_project_id_and_iid", unique: true, using: :btree
   add_index "milestones", ["title"], name: "index_milestones_on_title", using: :btree
   add_index "milestones", ["title"], name: "index_milestones_on_title_trigram", using: :gin, opclasses: {"title"=>"gin_trgm_ops"}
@@ -971,6 +1017,7 @@ ActiveRecord::Schema.define(version: 20170630111158) do
   end
 
   add_index "pages_domains", ["domain"], name: "index_pages_domains_on_domain", unique: true, using: :btree
+  add_index "pages_domains", ["project_id"], name: "index_pages_domains_on_project_id", using: :btree
 
   create_table "personal_access_tokens", force: :cascade do |t|
     t.integer "user_id", null: false
@@ -1020,6 +1067,7 @@ ActiveRecord::Schema.define(version: 20170630111158) do
   end
 
   add_index "project_group_links", ["group_id"], name: "index_project_group_links_on_group_id", using: :btree
+  add_index "project_group_links", ["project_id"], name: "index_project_group_links_on_project_id", using: :btree
 
   create_table "project_import_data", force: :cascade do |t|
     t.integer "project_id"
@@ -1086,6 +1134,7 @@ ActiveRecord::Schema.define(version: 20170630111158) do
     t.string "import_jid"
     t.integer "cached_markdown_version"
     t.datetime "last_repository_updated_at"
+    t.string "ci_config_path"
   end
 
   add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
@@ -1527,48 +1576,79 @@ ActiveRecord::Schema.define(version: 20170630111158) do
   add_index "web_hooks", ["project_id"], name: "index_web_hooks_on_project_id", using: :btree
   add_index "web_hooks", ["type"], name: "index_web_hooks_on_type", using: :btree
 
-  add_foreign_key "boards", "projects"
+  add_foreign_key "boards", "projects", name: "fk_f15266b5f9", on_delete: :cascade
   add_foreign_key "chat_teams", "namespaces", on_delete: :cascade
   add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify
   add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade
+  add_foreign_key "ci_builds", "projects", name: "fk_befce0568a", on_delete: :cascade
+  add_foreign_key "ci_pipeline_schedule_variables", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_41c35fda51", on_delete: :cascade
+  add_foreign_key "ci_group_variables", "namespaces", column: "group_id", name: "fk_33ae4d58d8", on_delete: :cascade
   add_foreign_key "ci_pipeline_schedules", "projects", name: "fk_8ead60fcc4", on_delete: :cascade
   add_foreign_key "ci_pipeline_schedules", "users", column: "owner_id", name: "fk_9ea99f58d2", on_delete: :nullify
   add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify
   add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify
+  add_foreign_key "ci_pipelines", "projects", name: "fk_86635dbd80", on_delete: :cascade
+  add_foreign_key "ci_runner_projects", "projects", name: "fk_4478a6f1e4", on_delete: :cascade
   add_foreign_key "ci_stages", "ci_pipelines", column: "pipeline_id", name: "fk_fb57e6cc56", on_delete: :cascade
   add_foreign_key "ci_stages", "projects", name: "fk_2360681d1d", on_delete: :cascade
   add_foreign_key "ci_trigger_requests", "ci_triggers", column: "trigger_id", name: "fk_b8ec8b7245", on_delete: :cascade
+  add_foreign_key "ci_triggers", "projects", name: "fk_e3e63f966e", on_delete: :cascade
   add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade
   add_foreign_key "ci_variables", "projects", name: "fk_ada5eb64b3", on_delete: :cascade
   add_foreign_key "container_repositories", "projects"
+  add_foreign_key "deploy_keys_projects", "projects", name: "fk_58a901ca7e", on_delete: :cascade
+  add_foreign_key "deployments", "projects", name: "fk_b9a3851b82", on_delete: :cascade
+  add_foreign_key "environments", "projects", name: "fk_d1c8c1da6a", on_delete: :cascade
+  add_foreign_key "events", "projects", name: "fk_0434b48643", on_delete: :cascade
+  add_foreign_key "forked_project_links", "projects", column: "forked_to_project_id", name: "fk_434510edb0", on_delete: :cascade
   add_foreign_key "issue_assignees", "issues", name: "fk_b7d881734a", on_delete: :cascade
   add_foreign_key "issue_assignees", "users", name: "fk_5e0c8d9154", on_delete: :cascade
   add_foreign_key "issue_metrics", "issues", on_delete: :cascade
+  add_foreign_key "issues", "projects", name: "fk_899c8f3231", on_delete: :cascade
   add_foreign_key "label_priorities", "labels", on_delete: :cascade
   add_foreign_key "label_priorities", "projects", on_delete: :cascade
   add_foreign_key "labels", "namespaces", column: "group_id", on_delete: :cascade
-  add_foreign_key "lists", "boards"
-  add_foreign_key "lists", "labels"
+  add_foreign_key "labels", "projects", name: "fk_7de4989a69", on_delete: :cascade
+  add_foreign_key "lists", "boards", name: "fk_0d3f677137", on_delete: :cascade
+  add_foreign_key "lists", "labels", name: "fk_7a5553d60f", on_delete: :cascade
+  add_foreign_key "merge_request_diff_commits", "merge_request_diffs", on_delete: :cascade
   add_foreign_key "merge_request_diff_files", "merge_request_diffs", on_delete: :cascade
+  add_foreign_key "merge_request_diffs", "merge_requests", name: "fk_8483f3258f", on_delete: :cascade
   add_foreign_key "merge_request_metrics", "ci_pipelines", column: "pipeline_id", on_delete: :cascade
   add_foreign_key "merge_request_metrics", "merge_requests", on_delete: :cascade
+  add_foreign_key "merge_requests", "projects", column: "target_project_id", name: "fk_a6963e8447", on_delete: :cascade
   add_foreign_key "merge_requests_closing_issues", "issues", on_delete: :cascade
   add_foreign_key "merge_requests_closing_issues", "merge_requests", on_delete: :cascade
+  add_foreign_key "milestones", "namespaces", column: "group_id", name: "fk_95650a40d4", on_delete: :cascade
+  add_foreign_key "milestones", "projects", name: "fk_9bd0a0c791", on_delete: :cascade
+  add_foreign_key "notes", "projects", name: "fk_99e097b079", on_delete: :cascade
   add_foreign_key "oauth_openid_requests", "oauth_access_grants", column: "access_grant_id", name: "fk_oauth_openid_requests_oauth_access_grants_access_grant_id"
+  add_foreign_key "pages_domains", "projects", name: "fk_ea2f6dfc6f", on_delete: :cascade
   add_foreign_key "personal_access_tokens", "users"
   add_foreign_key "project_authorizations", "projects", on_delete: :cascade
   add_foreign_key "project_authorizations", "users", on_delete: :cascade
+  add_foreign_key "project_features", "projects", name: "fk_18513d9b92", on_delete: :cascade
+  add_foreign_key "project_group_links", "projects", name: "fk_daa8cee94c", on_delete: :cascade
+  add_foreign_key "project_import_data", "projects", name: "fk_ffb9ee3a10", on_delete: :cascade
   add_foreign_key "project_statistics", "projects", on_delete: :cascade
-  add_foreign_key "protected_branch_merge_access_levels", "protected_branches"
-  add_foreign_key "protected_branch_push_access_levels", "protected_branches"
+  add_foreign_key "protected_branch_merge_access_levels", "protected_branches", name: "fk_8a3072ccb3", on_delete: :cascade
+  add_foreign_key "protected_branch_push_access_levels", "protected_branches", name: "fk_9ffc86a3d9", on_delete: :cascade
+  add_foreign_key "protected_branches", "projects", name: "fk_7a9c6d93e7", on_delete: :cascade
   add_foreign_key "protected_tag_create_access_levels", "namespaces", column: "group_id"
   add_foreign_key "protected_tag_create_access_levels", "protected_tags"
   add_foreign_key "protected_tag_create_access_levels", "users"
+  add_foreign_key "protected_tags", "projects", name: "fk_8e4af87648", on_delete: :cascade
+  add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade
+  add_foreign_key "services", "projects", name: "fk_71cce407f9", on_delete: :cascade
+  add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade
   add_foreign_key "subscriptions", "projects", on_delete: :cascade
   add_foreign_key "system_note_metadata", "notes", name: "fk_d83a918cb1", on_delete: :cascade
   add_foreign_key "timelogs", "issues", name: "fk_timelogs_issues_issue_id", on_delete: :cascade
   add_foreign_key "timelogs", "merge_requests", name: "fk_timelogs_merge_requests_merge_request_id", on_delete: :cascade
+  add_foreign_key "todos", "projects", name: "fk_45054f9c45", on_delete: :cascade
   add_foreign_key "trending_projects", "projects", on_delete: :cascade
   add_foreign_key "u2f_registrations", "users"
+  add_foreign_key "users_star_projects", "projects", name: "fk_22cd27ddfc", on_delete: :cascade
   add_foreign_key "web_hook_logs", "web_hooks", on_delete: :cascade
+  add_foreign_key "web_hooks", "projects", name: "fk_0c8ca6d9d1", on_delete: :cascade
 end