Merge branch 'master' into live-trace-v2

6 files changed, 155 insertions, 18 deletions

diff --git a/db/migrate/20161220141214_remove_dot_git_from_group_names.rb b/db/migrate/20161220141214_remove_dot_git_from_group_names.rb
index bddc234db25..17357b67ab7 100644
--- a/db/migrate/20161220141214_remove_dot_git_from_group_names.rb
+++ b/db/migrate/20161220141214_remove_dot_git_from_group_names.rb
@@ -59,17 +59,17 @@ class RemoveDotGitFromGroupNames < ActiveRecord::Migration
   end
 
   def move_namespace(group_id, path_was, path)
-    repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row|
-      Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path
+    repository_storages = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row|
+      row['repository_storage']
     end.compact
 
     # Move the namespace directory in all storages paths used by member projects
-    repository_storage_paths.each do |repository_storage_path|
+    repository_storages.each do |repository_storage|
       # Ensure old directory exists before moving it
-      gitlab_shell.add_namespace(repository_storage_path, path_was)
+      gitlab_shell.add_namespace(repository_storage, path_was)
 
-      unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
-        Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}"
+      unless gitlab_shell.mv_namespace(repository_storage, path_was, path)
+        Rails.logger.error "Exception moving on shard #{repository_storage} from #{path_was} to #{path}"
 
         # if we cannot move namespace directory we should rollback
         # db changes in order to prevent out of sync between db and fs
diff --git a/db/migrate/20161226122833_remove_dot_git_from_usernames.rb b/db/migrate/20161226122833_remove_dot_git_from_usernames.rb
index 7c28d934c29..8986cd8cb4b 100644
--- a/db/migrate/20161226122833_remove_dot_git_from_usernames.rb
+++ b/db/migrate/20161226122833_remove_dot_git_from_usernames.rb
@@ -53,8 +53,8 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
     select_all("SELECT id, path FROM routes WHERE path = '#{quote_string(path)}'").present?
   end
 
-  def path_exists?(path, repository_storage_path)
-    repository_storage_path && gitlab_shell.exists?(repository_storage_path, path)
+  def path_exists?(shard, repository_storage_path)
+    repository_storage_path && gitlab_shell.exists?(shard, repository_storage_path)
   end
 
   # Accepts invalid path like test.git and returns test_git or
@@ -70,8 +70,8 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
   def check_routes(base, counter, path)
     route_exists = route_exists?(path)
 
-    Gitlab.config.repositories.storages.each_value do |storage|
-      if route_exists || path_exists?(path, storage.legacy_disk_path)
+    Gitlab.config.repositories.storages.each do |shard, storage|
+      if route_exists || path_exists?(shard, storage.legacy_disk_path)
         counter += 1
         path = "#{base}#{counter}"
 
@@ -83,17 +83,17 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
   end
 
   def move_namespace(namespace_id, path_was, path)
-    repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row|
-      Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path
+    repository_storages = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row|
+      row['repository_storage']
     end.compact
 
-    # Move the namespace directory in all storages paths used by member projects
-    repository_storage_paths.each do |repository_storage_path|
+    # Move the namespace directory in all storages used by member projects
+    repository_storages.each do |repository_storage|
       # Ensure old directory exists before moving it
-      gitlab_shell.add_namespace(repository_storage_path, path_was)
+      gitlab_shell.add_namespace(repository_storage, path_was)
 
-      unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
-        Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}"
+      unless gitlab_shell.mv_namespace(repository_storage, path_was, path)
+        Rails.logger.error "Exception moving on shard #{repository_storage} from #{path_was} to #{path}"
 
         # if we cannot move namespace directory we should rollback
         # db changes in order to prevent out of sync between db and fs
diff --git a/db/migrate/20180403035759_create_project_ci_cd_settings.rb b/db/migrate/20180403035759_create_project_ci_cd_settings.rb
new file mode 100644
index 00000000000..06856af6204
--- /dev/null
+++ b/db/migrate/20180403035759_create_project_ci_cd_settings.rb
@@ -0,0 +1,68 @@
+class CreateProjectCiCdSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    unless table_exists?(:project_ci_cd_settings)
+      create_table(:project_ci_cd_settings) do |t|
+        t.integer(:project_id, null: false)
+        t.boolean(:group_runners_enabled, default: true, null: false)
+      end
+    end
+
+    disable_statement_timeout
+
+    # This particular INSERT will take between 10 and 20 seconds.
+    execute 'INSERT INTO project_ci_cd_settings (project_id) SELECT id FROM projects'
+
+    # We add the index and foreign key separately so the above INSERT statement
+    # takes as little time as possible.
+    add_concurrent_index(:project_ci_cd_settings, :project_id, unique: true)
+
+    add_foreign_key_with_retry
+  end
+
+  def down
+    drop_table :project_ci_cd_settings
+  end
+
+  def add_foreign_key_with_retry
+    if Gitlab::Database.mysql?
+      # When using MySQL we don't support online upgrades, thus projects can't
+      # be deleted while we are running this migration.
+      return add_project_id_foreign_key
+    end
+
+    # Between the initial INSERT and the addition of the foreign key some
+    # projects may have been removed, leaving orphaned rows in our new settings
+    # table.
+    loop do
+      remove_orphaned_settings
+
+      begin
+        add_project_id_foreign_key
+        break
+      rescue ActiveRecord::InvalidForeignKey
+        say 'project_ci_cd_settings contains some orphaned rows, retrying...'
+      end
+    end
+  end
+
+  def add_project_id_foreign_key
+    add_concurrent_foreign_key(:project_ci_cd_settings, :projects, column: :project_id)
+  end
+
+  def remove_orphaned_settings
+    execute <<~SQL
+      DELETE FROM project_ci_cd_settings
+      WHERE NOT EXISTS (
+        SELECT 1
+        FROM projects
+        WHERE projects.id = project_ci_cd_settings.project_id
+      )
+    SQL
+  end
+end
diff --git a/db/migrate/20180425131009_assure_commits_count_for_merge_request_diff.rb b/db/migrate/20180425131009_assure_commits_count_for_merge_request_diff.rb
new file mode 100644
index 00000000000..0e991c23bfa
--- /dev/null
+++ b/db/migrate/20180425131009_assure_commits_count_for_merge_request_diff.rb
@@ -0,0 +1,27 @@
+class AssureCommitsCountForMergeRequestDiff < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class MergeRequestDiff < ActiveRecord::Base
+    self.table_name = 'merge_request_diffs'
+
+    include ::EachBatch
+  end
+
+  def up
+    Gitlab::BackgroundMigration.steal('AddMergeRequestDiffCommitsCount')
+
+    MergeRequestDiff.where(commits_count: nil).each_batch(of: 50) do |batch|
+      range = batch.pluck('MIN(id)', 'MAX(id)').first
+
+      Gitlab::BackgroundMigration::AddMergeRequestDiffCommitsCount.new.perform(*range)
+    end
+  end
+
+  def down
+    # noop
+  end
+end
diff --git a/db/post_migrate/20180409170809_populate_missing_project_ci_cd_settings.rb b/db/post_migrate/20180409170809_populate_missing_project_ci_cd_settings.rb
new file mode 100644
index 00000000000..3b0fdb3aeea
--- /dev/null
+++ b/db/post_migrate/20180409170809_populate_missing_project_ci_cd_settings.rb
@@ -0,0 +1,34 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class PopulateMissingProjectCiCdSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    # MySQL does not support online upgrades, thus there can't be any missing
+    # rows.
+    return if Gitlab::Database.mysql?
+
+    # Projects created after the initial migration but before the code started
+    # using ProjectCiCdSetting won't have a corresponding row in
+    # project_ci_cd_settings, so let's fix that.
+    execute <<~SQL
+      INSERT INTO project_ci_cd_settings (project_id)
+      SELECT id
+      FROM projects
+      WHERE NOT EXISTS (
+        SELECT 1
+        FROM project_ci_cd_settings
+        WHERE project_ci_cd_settings.project_id = projects.id
+      )
+    SQL
+  end
+
+  def down
+    # There's nothing to revert for this migration.
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 6e459b2c286..19ae9cbf443 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180418053107) do
+ActiveRecord::Schema.define(version: 20180425131009) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1445,6 +1445,13 @@ ActiveRecord::Schema.define(version: 20180418053107) do
 
   add_index "project_auto_devops", ["project_id"], name: "index_project_auto_devops_on_project_id", unique: true, using: :btree
 
+  create_table "project_ci_cd_settings", force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.boolean "group_runners_enabled", default: true, null: false
+  end
+
+  add_index "project_ci_cd_settings", ["project_id"], name: "index_project_ci_cd_settings_on_project_id", unique: true, using: :btree
+
   create_table "project_custom_attributes", force: :cascade do |t|
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
@@ -2172,6 +2179,7 @@ ActiveRecord::Schema.define(version: 20180418053107) do
   add_foreign_key "project_authorizations", "projects", on_delete: :cascade
   add_foreign_key "project_authorizations", "users", on_delete: :cascade
   add_foreign_key "project_auto_devops", "projects", on_delete: :cascade
+  add_foreign_key "project_ci_cd_settings", "projects", name: "fk_24c15d2f2e", on_delete: :cascade
   add_foreign_key "project_custom_attributes", "projects", on_delete: :cascade
   add_foreign_key "project_deploy_tokens", "deploy_tokens", on_delete: :cascade
   add_foreign_key "project_deploy_tokens", "projects", on_delete: :cascade