Merge branch 'master' into sh-headless-chrome-support

37 files changed, 872 insertions, 0 deletions

diff --git a/db/migrate/20160615191922_set_missing_stage_on_ci_builds.rb b/db/migrate/20160615191922_set_missing_stage_on_ci_builds.rb
index 4d6a61bd614..5336b036bca 100644
--- a/db/migrate/20160615191922_set_missing_stage_on_ci_builds.rb
+++ b/db/migrate/20160615191922_set_missing_stage_on_ci_builds.rb
@@ -2,6 +2,8 @@
 class SetMissingStageOnCiBuilds < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
+  disable_ddl_transaction!
+
   def up
     update_column_in_batches(:ci_builds, :stage, :test) do |table, query|
       query.where(table[:stage].eq(nil))
diff --git a/db/migrate/20160721081015_drop_and_readd_has_external_wiki_in_projects.rb b/db/migrate/20160721081015_drop_and_readd_has_external_wiki_in_projects.rb
index b2a2ce41391..abe8e701e23 100644
--- a/db/migrate/20160721081015_drop_and_readd_has_external_wiki_in_projects.rb
+++ b/db/migrate/20160721081015_drop_and_readd_has_external_wiki_in_projects.rb
@@ -5,6 +5,8 @@ class DropAndReaddHasExternalWikiInProjects < ActiveRecord::Migration
   # Set this constant to true if this migration requires downtime.
   DOWNTIME = false
 
+  disable_ddl_transaction!
+
   def up
     update_column_in_batches(:projects, :has_external_wiki, nil) do |table, query|
       query.where(table[:has_external_wiki].not_eq(nil))
diff --git a/db/migrate/20160804142904_add_ci_config_file_to_project.rb b/db/migrate/20160804142904_add_ci_config_file_to_project.rb
new file mode 100644
index 00000000000..341ae555c1b
--- /dev/null
+++ b/db/migrate/20160804142904_add_ci_config_file_to_project.rb
@@ -0,0 +1,11 @@
+class AddCiConfigFileToProject < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :projects, :ci_config_path, :string
+  end
+
+  def down
+    remove_column :projects, :ci_config_path
+  end
+end
diff --git a/db/migrate/20160901141443_set_confidential_issues_events_on_webhooks.rb b/db/migrate/20160901141443_set_confidential_issues_events_on_webhooks.rb
index febd2c0e65e..f8486e3e1a6 100644
--- a/db/migrate/20160901141443_set_confidential_issues_events_on_webhooks.rb
+++ b/db/migrate/20160901141443_set_confidential_issues_events_on_webhooks.rb
@@ -4,6 +4,8 @@ class SetConfidentialIssuesEventsOnWebhooks < ActiveRecord::Migration
 
   DOWNTIME = false
 
+  disable_ddl_transaction!
+
   def up
     update_column_in_batches(:web_hooks, :confidential_issues_events, true) do |table, query|
       query.where(table[:issues_events].eq(true))
diff --git a/db/migrate/20160919144305_add_type_to_labels.rb b/db/migrate/20160919144305_add_type_to_labels.rb
index 2d2725ccf59..d08b339cd27 100644
--- a/db/migrate/20160919144305_add_type_to_labels.rb
+++ b/db/migrate/20160919144305_add_type_to_labels.rb
@@ -5,6 +5,8 @@ class AddTypeToLabels < ActiveRecord::Migration
   DOWNTIME = true
   DOWNTIME_REASON = 'Labels will not work as expected until this migration is complete.'
 
+  disable_ddl_transaction!
+
   def change
     add_column :labels, :type, :string
 
diff --git a/db/migrate/20161018124658_make_project_owners_masters.rb b/db/migrate/20161018124658_make_project_owners_masters.rb
index fe11699c196..cb93b449067 100644
--- a/db/migrate/20161018124658_make_project_owners_masters.rb
+++ b/db/migrate/20161018124658_make_project_owners_masters.rb
@@ -4,6 +4,8 @@ class MakeProjectOwnersMasters < ActiveRecord::Migration
 
   DOWNTIME = false
 
+  disable_ddl_transaction!
+
   def up
     update_column_in_batches(:members, :access_level, 40) do |table, query|
       query.where(table[:access_level].eq(50).and(table[:source_type].eq('Project')))
diff --git a/db/migrate/20161227192806_rename_slack_and_mattermost_notification_services.rb b/db/migrate/20161227192806_rename_slack_and_mattermost_notification_services.rb
index c7cada6dfc5..6b15e5caccf 100644
--- a/db/migrate/20161227192806_rename_slack_and_mattermost_notification_services.rb
+++ b/db/migrate/20161227192806_rename_slack_and_mattermost_notification_services.rb
@@ -4,6 +4,8 @@ class RenameSlackAndMattermostNotificationServices < ActiveRecord::Migration
 
   DOWNTIME = false
 
+  disable_ddl_transaction!
+
   def up
     update_column_in_batches(:services, :type, 'SlackService') do |table, query|
       query.where(table[:type].eq('SlackNotificationService'))
diff --git a/db/migrate/20170222111732_create_gpg_keys.rb b/db/migrate/20170222111732_create_gpg_keys.rb
new file mode 100644
index 00000000000..541228e8735
--- /dev/null
+++ b/db/migrate/20170222111732_create_gpg_keys.rb
@@ -0,0 +1,19 @@
+class CreateGpgKeys < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    create_table :gpg_keys do |t|
+      t.timestamps_with_timezone null: false
+
+      t.references :user, index: true, foreign_key: { on_delete: :cascade }
+
+      t.binary :primary_keyid
+      t.binary :fingerprint
+
+      t.text :key
+
+      t.index :primary_keyid, unique: true, length: Gitlab::Database.mysql? ? 20 : nil
+      t.index :fingerprint, unique: true, length: Gitlab::Database.mysql? ? 20 : nil
+    end
+  end
+end
diff --git a/db/migrate/20170428064307_add_column_delete_error_to_projects.rb b/db/migrate/20170428064307_add_column_delete_error_to_projects.rb
new file mode 100644
index 00000000000..09f9d9b5b7a
--- /dev/null
+++ b/db/migrate/20170428064307_add_column_delete_error_to_projects.rb
@@ -0,0 +1,7 @@
+class AddColumnDeleteErrorToProjects < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :projects, :delete_error, :text
+  end
+end
diff --git a/db/migrate/20170525130346_create_group_variables_table.rb b/db/migrate/20170525130346_create_group_variables_table.rb
new file mode 100644
index 00000000000..eaa38dbc40d
--- /dev/null
+++ b/db/migrate/20170525130346_create_group_variables_table.rb
@@ -0,0 +1,23 @@
+class CreateGroupVariablesTable < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    create_table :ci_group_variables do |t|
+      t.string :key, null: false
+      t.text :value
+      t.text :encrypted_value
+      t.string :encrypted_value_salt
+      t.string :encrypted_value_iv
+      t.integer :group_id, null: false
+      t.boolean :protected, default: false, null: false
+
+      t.timestamps_with_timezone null: false
+    end
+
+    add_index :ci_group_variables, [:group_id, :key], unique: true
+  end
+
+  def down
+    drop_table :ci_group_variables
+  end
+end
diff --git a/db/migrate/20170525130758_add_foreign_key_to_group_variables.rb b/db/migrate/20170525130758_add_foreign_key_to_group_variables.rb
new file mode 100644
index 00000000000..0146235c5ba
--- /dev/null
+++ b/db/migrate/20170525130758_add_foreign_key_to_group_variables.rb
@@ -0,0 +1,15 @@
+class AddForeignKeyToGroupVariables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :ci_group_variables, :namespaces, column: :group_id
+  end
+
+  def down
+    remove_foreign_key :ci_group_variables, column: :group_id
+  end
+end
diff --git a/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb b/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb
new file mode 100644
index 00000000000..3eaafac321d
--- /dev/null
+++ b/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb
@@ -0,0 +1,187 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  CONCURRENCY = 4
+
+  disable_ddl_transaction!
+
+  # The tables/columns for which to remove orphans and add foreign keys. Order
+  # matters as some tables/columns should be processed before others.
+  TABLES = [
+    [:boards, :projects, :project_id],
+    [:lists, :labels, :label_id],
+    [:lists, :boards, :board_id],
+    [:services, :projects, :project_id],
+    [:forked_project_links, :projects, :forked_to_project_id],
+    [:merge_requests, :projects, :target_project_id],
+    [:labels, :projects, :project_id],
+    [:issues, :projects, :project_id],
+    [:events, :projects, :project_id],
+    [:milestones, :projects, :project_id],
+    [:notes, :projects, :project_id],
+    [:snippets, :projects, :project_id],
+    [:web_hooks, :projects, :project_id],
+    [:protected_branch_merge_access_levels, :protected_branches, :protected_branch_id],
+    [:protected_branch_push_access_levels, :protected_branches, :protected_branch_id],
+    [:protected_branches, :projects, :project_id],
+    [:protected_tags, :projects, :project_id],
+    [:deploy_keys_projects, :projects, :project_id],
+    [:users_star_projects, :projects, :project_id],
+    [:releases, :projects, :project_id],
+    [:project_group_links, :projects, :project_id],
+    [:pages_domains, :projects, :project_id],
+    [:todos, :projects, :project_id],
+    [:project_import_data, :projects, :project_id],
+    [:project_features, :projects, :project_id],
+    [:ci_builds, :projects, :project_id],
+    [:ci_pipelines, :projects, :project_id],
+    [:ci_runner_projects, :projects, :project_id],
+    [:ci_triggers, :projects, :project_id],
+    [:environments, :projects, :project_id],
+    [:deployments, :projects, :project_id]
+  ]
+
+  def up
+    # These existing foreign keys don't have an "ON DELETE CASCADE" clause.
+    remove_foreign_key_without_error(:boards, :project_id)
+    remove_foreign_key_without_error(:lists, :label_id)
+    remove_foreign_key_without_error(:lists, :board_id)
+    remove_foreign_key_without_error(:protected_branch_merge_access_levels,
+                                     :protected_branch_id)
+
+    remove_foreign_key_without_error(:protected_branch_push_access_levels,
+                                     :protected_branch_id)
+
+    remove_orphaned_rows
+    add_foreign_keys
+
+    # These columns are not indexed yet, meaning a cascading delete would take
+    # forever.
+    add_concurrent_index(:project_group_links, :project_id)
+    add_concurrent_index(:pages_domains, :project_id)
+  end
+
+  def down
+    TABLES.each do |(source, _, column)|
+      remove_foreign_key_without_error(source, column)
+    end
+
+    add_concurrent_foreign_key(:boards, :projects, column: :project_id)
+    add_concurrent_foreign_key(:lists, :labels, column: :label_id)
+    add_concurrent_foreign_key(:lists, :boards, column: :board_id)
+
+    add_concurrent_foreign_key(:protected_branch_merge_access_levels,
+                               :protected_branches,
+                               column: :protected_branch_id)
+
+    add_concurrent_foreign_key(:protected_branch_push_access_levels,
+                               :protected_branches,
+                               column: :protected_branch_id)
+
+    remove_index_without_error(:project_group_links, :project_id)
+    remove_index_without_error(:pages_domains, :project_id)
+  end
+
+  def add_foreign_keys
+    TABLES.each do |(source, target, column)|
+      add_concurrent_foreign_key(source, target, column: column)
+    end
+  end
+
+  # Removes orphans from various tables concurrently.
+  def remove_orphaned_rows
+    Gitlab::Database.with_connection_pool(CONCURRENCY) do |pool|
+      queues = queues_for_rows(TABLES)
+
+      threads = queues.map do |queue|
+        Thread.new do
+          pool.with_connection do |connection|
+            Thread.current[:foreign_key_connection] = connection
+
+            # Disables statement timeouts for the current connection. This is
+            # necessary as removing of orphaned data might otherwise exceed the
+            # statement timeout.
+            disable_statement_timeout
+
+            remove_orphans(*queue.pop) until queue.empty?
+
+            steal_from_queues(queues - [queue])
+          end
+        end
+      end
+
+      threads.each(&:join)
+    end
+  end
+
+  def steal_from_queues(queues)
+    loop do
+      stolen = false
+
+      queues.each do |queue|
+        # Stealing is racy so it's possible a pop might be called on an
+        # already-empty queue.
+        begin
+          remove_orphans(*queue.pop(true))
+          stolen = true
+        rescue ThreadError
+        end
+      end
+
+      break unless stolen
+    end
+  end
+
+  def remove_orphans(source, target, column)
+    quoted_source = quote_table_name(source)
+    quoted_target = quote_table_name(target)
+    quoted_column = quote_column_name(column)
+
+    execute <<-EOF.strip_heredoc
+    DELETE FROM #{quoted_source}
+    WHERE NOT EXISTS (
+      SELECT true
+      FROM #{quoted_target}
+      WHERE #{quoted_target}.id = #{quoted_source}.#{quoted_column}
+    )
+    AND #{quoted_source}.#{quoted_column} IS NOT NULL
+    EOF
+  end
+
+  def remove_foreign_key_without_error(table, column)
+    remove_foreign_key(table, column: column)
+  rescue ArgumentError
+  end
+
+  def remove_index_without_error(table, column)
+    remove_concurrent_index(table, column)
+  rescue ArgumentError
+  end
+
+  def connection
+    # Rails memoizes connection objects, but this causes them to be shared
+    # amongst threads; we don't want that.
+    Thread.current[:foreign_key_connection] || ActiveRecord::Base.connection
+  end
+
+  def queues_for_rows(rows)
+    queues = Array.new(CONCURRENCY) { Queue.new }
+    slice_size = rows.length / CONCURRENCY
+
+    # Divide all the tuples as evenly as possible amongst the queues.
+    rows.each_slice(slice_size).each_with_index do |slice, index|
+      bucket = index % CONCURRENCY
+
+      slice.each do |row|
+        queues[bucket] << row
+      end
+    end
+
+    queues
+  end
+end
diff --git a/db/migrate/20170613154149_create_gpg_signatures.rb b/db/migrate/20170613154149_create_gpg_signatures.rb
new file mode 100644
index 00000000000..f6b5e7ebb7b
--- /dev/null
+++ b/db/migrate/20170613154149_create_gpg_signatures.rb
@@ -0,0 +1,23 @@
+class CreateGpgSignatures < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    create_table :gpg_signatures do |t|
+      t.timestamps_with_timezone null: false
+
+      t.references :project, index: true, foreign_key: { on_delete: :cascade }
+      t.references :gpg_key, index: true, foreign_key: { on_delete: :nullify }
+
+      t.boolean :valid_signature
+
+      t.binary :commit_sha
+      t.binary :gpg_key_primary_keyid
+
+      t.text :gpg_key_user_name
+      t.text :gpg_key_user_email
+
+      t.index :commit_sha, unique: true, length: Gitlab::Database.mysql? ? 20 : nil
+      t.index :gpg_key_primary_keyid, length: Gitlab::Database.mysql? ? 20 : nil
+    end
+  end
+end
diff --git a/db/migrate/20170616133147_create_merge_request_diff_commits.rb b/db/migrate/20170616133147_create_merge_request_diff_commits.rb
new file mode 100644
index 00000000000..616464cb470
--- /dev/null
+++ b/db/migrate/20170616133147_create_merge_request_diff_commits.rb
@@ -0,0 +1,20 @@
+class CreateMergeRequestDiffCommits < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    create_table :merge_request_diff_commits, id: false do |t|
+      t.datetime_with_timezone :authored_date
+      t.datetime_with_timezone :committed_date
+      t.belongs_to :merge_request_diff, null: false, foreign_key: { on_delete: :cascade }
+      t.integer :relative_order, null: false
+      t.binary :sha, null: false, limit: 20
+      t.text :author_name
+      t.text :author_email
+      t.text :committer_name
+      t.text :committer_email
+      t.text :message
+
+      t.index [:merge_request_diff_id, :relative_order], name: 'index_merge_request_diff_commits_on_mr_diff_id_and_order', unique: true
+    end
+  end
+end
diff --git a/db/migrate/20170620064728_create_ci_pipeline_schedule_variables.rb b/db/migrate/20170620064728_create_ci_pipeline_schedule_variables.rb
new file mode 100644
index 00000000000..92833765a82
--- /dev/null
+++ b/db/migrate/20170620064728_create_ci_pipeline_schedule_variables.rb
@@ -0,0 +1,25 @@
+class CreateCiPipelineScheduleVariables < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    create_table :ci_pipeline_schedule_variables do |t|
+      t.string :key, null: false
+      t.text :value
+      t.text :encrypted_value
+      t.string :encrypted_value_salt
+      t.string :encrypted_value_iv
+      t.integer :pipeline_schedule_id, null: false
+
+      t.timestamps_with_timezone null: true
+    end
+
+    add_index :ci_pipeline_schedule_variables,
+      [:pipeline_schedule_id, :key],
+      name: "index_ci_pipeline_schedule_variables_on_schedule_id_and_key",
+      unique: true
+  end
+
+  def down
+    drop_table :ci_pipeline_schedule_variables
+  end
+end
diff --git a/db/migrate/20170620065449_add_foreign_key_to_ci_pipeline_schedule_variables.rb b/db/migrate/20170620065449_add_foreign_key_to_ci_pipeline_schedule_variables.rb
new file mode 100644
index 00000000000..7bbf66e0ac3
--- /dev/null
+++ b/db/migrate/20170620065449_add_foreign_key_to_ci_pipeline_schedule_variables.rb
@@ -0,0 +1,15 @@
+class AddForeignKeyToCiPipelineScheduleVariables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key(:ci_pipeline_schedule_variables, :ci_pipeline_schedules, column: :pipeline_schedule_id)
+  end
+
+  def down
+    remove_foreign_key(:ci_pipeline_schedule_variables, column: :pipeline_schedule_id)
+  end
+end
diff --git a/db/migrate/20170622130029_correct_protected_branches_foreign_keys.rb b/db/migrate/20170622130029_correct_protected_branches_foreign_keys.rb
new file mode 100644
index 00000000000..46497775527
--- /dev/null
+++ b/db/migrate/20170622130029_correct_protected_branches_foreign_keys.rb
@@ -0,0 +1,40 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CorrectProtectedBranchesForeignKeys < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    remove_foreign_key_without_error(:protected_branch_push_access_levels,
+                                     column: :protected_branch_id)
+
+    execute <<-EOF
+    DELETE FROM protected_branch_push_access_levels
+    WHERE NOT EXISTS (
+      SELECT true
+      FROM protected_branches
+      WHERE protected_branch_push_access_levels.protected_branch_id = protected_branches.id
+    )
+    AND protected_branch_id IS NOT NULL
+    EOF
+
+    add_concurrent_foreign_key(:protected_branch_push_access_levels,
+                               :protected_branches,
+                               column: :protected_branch_id)
+  end
+
+  def down
+    # Previously there was a foreign key without a CASCADING DELETE, so we'll
+    # just leave the foreign key in place.
+  end
+
+  def remove_foreign_key_without_error(*args)
+    remove_foreign_key(*args)
+  rescue ArgumentError
+  end
+end
diff --git a/db/migrate/20170622132212_add_foreign_key_for_merge_request_diffs.rb b/db/migrate/20170622132212_add_foreign_key_for_merge_request_diffs.rb
new file mode 100644
index 00000000000..9f524fac8a7
--- /dev/null
+++ b/db/migrate/20170622132212_add_foreign_key_for_merge_request_diffs.rb
@@ -0,0 +1,30 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddForeignKeyForMergeRequestDiffs < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    execute <<-EOF
+    DELETE FROM merge_request_diffs
+    WHERE NOT EXISTS (
+      SELECT true
+      FROM merge_requests
+      WHERE merge_requests.id = merge_request_diffs.merge_request_id
+    )
+    EOF
+
+    add_concurrent_foreign_key(:merge_request_diffs,
+                               :merge_requests,
+                               column: :merge_request_id)
+  end
+
+  def down
+    remove_foreign_key(:merge_request_diffs, column: :merge_request_id)
+  end
+end
diff --git a/db/migrate/20170622135451_rename_duplicated_variable_key.rb b/db/migrate/20170622135451_rename_duplicated_variable_key.rb
new file mode 100644
index 00000000000..368718ab0ce
--- /dev/null
+++ b/db/migrate/20170622135451_rename_duplicated_variable_key.rb
@@ -0,0 +1,38 @@
+class RenameDuplicatedVariableKey < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    execute(<<~SQL)
+      UPDATE ci_variables
+      SET #{key} = CONCAT(#{key}, #{underscore}, id)
+      WHERE id IN (
+        SELECT *
+        FROM ( -- MySQL requires an extra layer
+          SELECT dup.id
+          FROM ci_variables dup
+          INNER JOIN (SELECT max(id) AS id, #{key}, project_id
+                      FROM ci_variables tmp
+                      GROUP BY #{key}, project_id) var
+          USING (#{key}, project_id) where dup.id <> var.id
+        ) dummy
+      )
+    SQL
+  end
+
+  def down
+    # noop
+  end
+
+  def key
+    # key needs to be quoted in MySQL
+    quote_column_name('key')
+  end
+
+  def underscore
+    quote('_')
+  end
+end
diff --git a/db/migrate/20170622135628_add_environment_scope_to_ci_variables.rb b/db/migrate/20170622135628_add_environment_scope_to_ci_variables.rb
new file mode 100644
index 00000000000..17fe062d8d5
--- /dev/null
+++ b/db/migrate/20170622135628_add_environment_scope_to_ci_variables.rb
@@ -0,0 +1,15 @@
+class AddEnvironmentScopeToCiVariables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:ci_variables, :environment_scope, :string, default: '*')
+  end
+
+  def down
+    remove_column(:ci_variables, :environment_scope)
+  end
+end
diff --git a/db/migrate/20170622135728_add_unique_constraint_to_ci_variables.rb b/db/migrate/20170622135728_add_unique_constraint_to_ci_variables.rb
new file mode 100644
index 00000000000..8b2cc40ee59
--- /dev/null
+++ b/db/migrate/20170622135728_add_unique_constraint_to_ci_variables.rb
@@ -0,0 +1,38 @@
+class AddUniqueConstraintToCiVariables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    unless this_index_exists?
+      add_concurrent_index(:ci_variables, columns, name: index_name, unique: true)
+    end
+  end
+
+  def down
+    if this_index_exists?
+      if Gitlab::Database.mysql? && !index_exists?(:ci_variables, :project_id)
+        # Need to add this index for MySQL project_id foreign key constraint
+        add_concurrent_index(:ci_variables, :project_id)
+      end
+
+      remove_concurrent_index(:ci_variables, columns, name: index_name)
+    end
+  end
+
+  private
+
+  def this_index_exists?
+    index_exists?(:ci_variables, columns, name: index_name)
+  end
+
+  def columns
+    @columns ||= [:project_id, :key, :environment_scope]
+  end
+
+  def index_name
+    'index_ci_variables_on_project_id_and_key_and_environment_scope'
+  end
+end
diff --git a/db/migrate/20170622162730_add_ref_fetched_to_merge_request.rb b/db/migrate/20170622162730_add_ref_fetched_to_merge_request.rb
new file mode 100644
index 00000000000..62aa1a4b4f0
--- /dev/null
+++ b/db/migrate/20170622162730_add_ref_fetched_to_merge_request.rb
@@ -0,0 +1,9 @@
+class AddRefFetchedToMergeRequest < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :merge_requests, :ref_fetched, :boolean
+  end
+end
diff --git a/db/migrate/20170623080805_remove_ci_variables_project_id_index.rb b/db/migrate/20170623080805_remove_ci_variables_project_id_index.rb
new file mode 100644
index 00000000000..ddcc0292b9d
--- /dev/null
+++ b/db/migrate/20170623080805_remove_ci_variables_project_id_index.rb
@@ -0,0 +1,19 @@
+class RemoveCiVariablesProjectIdIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    if index_exists?(:ci_variables, :project_id)
+      remove_concurrent_index(:ci_variables, :project_id)
+    end
+  end
+
+  def down
+    unless index_exists?(:ci_variables, :project_id)
+      add_concurrent_index(:ci_variables, :project_id)
+    end
+  end
+end
diff --git a/db/migrate/20170629171610_rename_application_settings_signin_enabled_to_password_authentication_enabled.rb b/db/migrate/20170629171610_rename_application_settings_signin_enabled_to_password_authentication_enabled.rb
new file mode 100644
index 00000000000..858b3bebace
--- /dev/null
+++ b/db/migrate/20170629171610_rename_application_settings_signin_enabled_to_password_authentication_enabled.rb
@@ -0,0 +1,15 @@
+class RenameApplicationSettingsSigninEnabledToPasswordAuthenticationEnabled < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    rename_column_concurrently :application_settings, :signin_enabled, :password_authentication_enabled
+  end
+
+  def down
+    cleanup_concurrent_column_rename :application_settings, :password_authentication_enabled, :signin_enabled
+  end
+end
diff --git a/db/migrate/20170703102400_add_stage_id_foreign_key_to_builds.rb b/db/migrate/20170703102400_add_stage_id_foreign_key_to_builds.rb
new file mode 100644
index 00000000000..68b947583d3
--- /dev/null
+++ b/db/migrate/20170703102400_add_stage_id_foreign_key_to_builds.rb
@@ -0,0 +1,35 @@
+class AddStageIdForeignKeyToBuilds < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    unless index_exists?(:ci_builds, :stage_id)
+      add_concurrent_index(:ci_builds, :stage_id)
+    end
+
+    unless foreign_key_exists?(:ci_builds, :stage_id)
+      add_concurrent_foreign_key(:ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade)
+    end
+  end
+
+  def down
+    if foreign_key_exists?(:ci_builds, :stage_id)
+      remove_foreign_key(:ci_builds, column: :stage_id)
+    end
+
+    if index_exists?(:ci_builds, :stage_id)
+      remove_concurrent_index(:ci_builds, :stage_id)
+    end
+  end
+
+  private
+
+  def foreign_key_exists?(table, column)
+    foreign_keys(:ci_builds).any? do |key|
+      key.options[:column] == column.to_s
+    end
+  end
+end
diff --git a/db/migrate/20170706151212_add_performance_bar_allowed_group_id_to_application_settings.rb b/db/migrate/20170706151212_add_performance_bar_allowed_group_id_to_application_settings.rb
new file mode 100644
index 00000000000..fe9970ddc71
--- /dev/null
+++ b/db/migrate/20170706151212_add_performance_bar_allowed_group_id_to_application_settings.rb
@@ -0,0 +1,9 @@
+class AddPerformanceBarAllowedGroupIdToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :performance_bar_allowed_group_id, :integer
+  end
+end
diff --git a/db/migrate/20170707183807_add_group_id_to_milestones.rb b/db/migrate/20170707183807_add_group_id_to_milestones.rb
new file mode 100644
index 00000000000..675ffd4a1c9
--- /dev/null
+++ b/db/migrate/20170707183807_add_group_id_to_milestones.rb
@@ -0,0 +1,20 @@
+class AddGroupIdToMilestones < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    return if column_exists? :milestones, :group_id
+
+    change_column_null :milestones, :project_id, true
+
+    add_column :milestones, :group_id, :integer
+  end
+
+  def down
+    # We cannot rollback project_id not null constraint if there are records
+    # with null values.
+    execute "DELETE from milestones WHERE project_id IS NULL"
+
+    remove_column :milestones, :group_id
+    change_column :milestones, :project_id, :integer, null: false
+  end
+end
diff --git a/db/migrate/20170707184243_add_group_milestone_id_indexes.rb b/db/migrate/20170707184243_add_group_milestone_id_indexes.rb
new file mode 100644
index 00000000000..aa48fe90cad
--- /dev/null
+++ b/db/migrate/20170707184243_add_group_milestone_id_indexes.rb
@@ -0,0 +1,21 @@
+class AddGroupMilestoneIdIndexes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    return if index_exists?(:milestones, :group_id)
+
+    add_concurrent_foreign_key :milestones, :namespaces, column: :group_id, on_delete: :cascade
+
+    add_concurrent_index :milestones, :group_id
+  end
+
+  def down
+    remove_foreign_key :milestones, column: :group_id
+
+    remove_concurrent_index :milestones, :group_id
+  end
+end
diff --git a/db/migrate/20170707184244_remove_wrong_versions_from_schema_versions.rb b/db/migrate/20170707184244_remove_wrong_versions_from_schema_versions.rb
new file mode 100644
index 00000000000..38536a8b06a
--- /dev/null
+++ b/db/migrate/20170707184244_remove_wrong_versions_from_schema_versions.rb
@@ -0,0 +1,10 @@
+class RemoveWrongVersionsFromSchemaVersions < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    execute("DELETE FROM schema_migrations WHERE version IN ('20170723183807', '20170724184243')")
+  end
+
+  def down
+  end
+end
diff --git a/db/migrate/20170710083355_clean_stage_id_reference_migration.rb b/db/migrate/20170710083355_clean_stage_id_reference_migration.rb
new file mode 100644
index 00000000000..681203eaf40
--- /dev/null
+++ b/db/migrate/20170710083355_clean_stage_id_reference_migration.rb
@@ -0,0 +1,18 @@
+class CleanStageIdReferenceMigration < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  ##
+  # `MigrateStageIdReferenceInBackground` background migration cleanup.
+  #
+  def up
+    Gitlab::BackgroundMigration.steal('MigrateBuildStageIdReference')
+  end
+
+  def down
+    # noop
+  end
+end
diff --git a/db/migrate/20170713104829_add_foreign_key_to_merge_requests.rb b/db/migrate/20170713104829_add_foreign_key_to_merge_requests.rb
new file mode 100644
index 00000000000..c25d4fd5986
--- /dev/null
+++ b/db/migrate/20170713104829_add_foreign_key_to_merge_requests.rb
@@ -0,0 +1,45 @@
+class AddForeignKeyToMergeRequests < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class MergeRequest < ActiveRecord::Base
+    self.table_name = 'merge_requests'
+    include ::EachBatch
+  end
+
+  def up
+    scope = <<-SQL.strip_heredoc
+      head_pipeline_id IS NOT NULL
+        AND NOT EXISTS (
+          SELECT 1 FROM ci_pipelines
+            WHERE ci_pipelines.id = merge_requests.head_pipeline_id
+        )
+    SQL
+
+    MergeRequest.where(scope).each_batch(of: 1000) do |merge_requests|
+      merge_requests.update_all(head_pipeline_id: nil)
+    end
+
+    unless foreign_key_exists?(:merge_requests, :head_pipeline_id)
+      add_concurrent_foreign_key(:merge_requests, :ci_pipelines,
+                                 column: :head_pipeline_id, on_delete: :nullify)
+    end
+  end
+
+  def down
+    if foreign_key_exists?(:merge_requests, :head_pipeline_id)
+      remove_foreign_key(:merge_requests, column: :head_pipeline_id)
+    end
+  end
+
+  private
+
+  def foreign_key_exists?(table, column)
+    foreign_keys(table).any? do |key|
+      key.options[:column] == column.to_s
+    end
+  end
+end
diff --git a/db/migrate/20170717074009_move_system_upload_folder.rb b/db/migrate/20170717074009_move_system_upload_folder.rb
new file mode 100644
index 00000000000..cce31794115
--- /dev/null
+++ b/db/migrate/20170717074009_move_system_upload_folder.rb
@@ -0,0 +1,60 @@
+class MoveSystemUploadFolder < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    unless file_storage?
+      say 'Using object storage, no need to move.'
+      return
+    end
+
+    unless File.directory?(old_directory)
+      say "#{old_directory} doesn't exist, no need to move it."
+      return
+    end
+
+    FileUtils.mkdir_p(File.join(base_directory, '-'))
+
+    say "Moving #{old_directory} -> #{new_directory}"
+    FileUtils.mv(old_directory, new_directory)
+    FileUtils.ln_s(new_directory, old_directory)
+  end
+
+  def down
+    unless file_storage?
+      say 'Using object storage, no need to move.'
+      return
+    end
+
+    unless File.directory?(new_directory)
+      say "#{new_directory} doesn't exist, no need to move it."
+      return
+    end
+
+    if File.symlink?(old_directory)
+      say "Removing #{old_directory} -> #{new_directory} symlink"
+      FileUtils.rm(old_directory)
+    end
+
+    say "Moving #{new_directory} -> #{old_directory}"
+    FileUtils.mv(new_directory, old_directory)
+  end
+
+  def new_directory
+    File.join(base_directory, '-', 'system')
+  end
+
+  def old_directory
+    File.join(base_directory, 'system')
+  end
+
+  def base_directory
+    File.join(Rails.root, 'public', 'uploads')
+  end
+
+  def file_storage?
+    CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
+  end
+end
diff --git a/db/migrate/20170717200542_add_trusted_column_to_oauth_applications.rb b/db/migrate/20170717200542_add_trusted_column_to_oauth_applications.rb
new file mode 100644
index 00000000000..1a013e6aefb
--- /dev/null
+++ b/db/migrate/20170717200542_add_trusted_column_to_oauth_applications.rb
@@ -0,0 +1,15 @@
+class AddTrustedColumnToOauthApplications < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:oauth_applications, :trusted, :boolean, default: false)
+  end
+
+  def down
+    remove_column(:oauth_applications, :trusted)
+  end
+end
diff --git a/db/migrate/20170720130522_create_ci_pipeline_variables.rb b/db/migrate/20170720130522_create_ci_pipeline_variables.rb
new file mode 100644
index 00000000000..a784f5dd142
--- /dev/null
+++ b/db/migrate/20170720130522_create_ci_pipeline_variables.rb
@@ -0,0 +1,20 @@
+class CreateCiPipelineVariables < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    create_table :ci_pipeline_variables do |t|
+      t.string :key, null: false
+      t.text :value
+      t.text :encrypted_value
+      t.string :encrypted_value_salt
+      t.string :encrypted_value_iv
+      t.integer :pipeline_id, null: false
+    end
+
+    add_index :ci_pipeline_variables, [:pipeline_id, :key], unique: true
+  end
+
+  def down
+    drop_table :ci_pipeline_variables
+  end
+end
diff --git a/db/migrate/20170720130749_add_foreign_key_to_ci_pipeline_variables.rb b/db/migrate/20170720130749_add_foreign_key_to_ci_pipeline_variables.rb
new file mode 100644
index 00000000000..550b8a88f02
--- /dev/null
+++ b/db/migrate/20170720130749_add_foreign_key_to_ci_pipeline_variables.rb
@@ -0,0 +1,15 @@
+class AddForeignKeyToCiPipelineVariables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key(:ci_pipeline_variables, :ci_pipelines, column: :pipeline_id)
+  end
+
+  def down
+    remove_foreign_key(:ci_pipeline_variables, column: :pipeline_id)
+  end
+end
diff --git a/db/migrate/20170724214302_add_lower_path_index_to_redirect_routes.rb b/db/migrate/20170724214302_add_lower_path_index_to_redirect_routes.rb
new file mode 100644
index 00000000000..db60c2087b9
--- /dev/null
+++ b/db/migrate/20170724214302_add_lower_path_index_to_redirect_routes.rb
@@ -0,0 +1,34 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddLowerPathIndexToRedirectRoutes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'index_on_redirect_routes_lower_path'
+
+  disable_ddl_transaction!
+
+  def up
+    return unless Gitlab::Database.postgresql?
+
+    execute "CREATE INDEX CONCURRENTLY #{INDEX_NAME} ON redirect_routes (LOWER(path));"
+  end
+
+  def down
+    return unless Gitlab::Database.postgresql?
+
+    # Why not use remove_concurrent_index_by_name?
+    #
+    # `index_exists?` doesn't work on this index. Perhaps this is related to the
+    # fact that the index doesn't show up in the schema. And apparently it isn't
+    # trivial to write a query that checks for an index. BUT there is a
+    # convenient `IF EXISTS` parameter for `DROP INDEX`.
+    if supports_drop_index_concurrently?
+      disable_statement_timeout
+      execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME};"
+    else
+      execute "DROP INDEX IF EXISTS #{INDEX_NAME};"
+    end
+  end
+end
diff --git a/db/migrate/20170725145659_add_binary_to_merge_request_diff_files.rb b/db/migrate/20170725145659_add_binary_to_merge_request_diff_files.rb
new file mode 100644
index 00000000000..1f5fa7e3d49
--- /dev/null
+++ b/db/migrate/20170725145659_add_binary_to_merge_request_diff_files.rb
@@ -0,0 +1,9 @@
+class AddBinaryToMergeRequestDiffFiles < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :merge_request_diff_files, :binary, :boolean
+  end
+end