Merge branch 'master' into backstage/gb/rename-ci-cd-processing-sidekiq-queues

* master: (115 commits)
  Use event-based waiting in Gitlab::JobWaiter
  Make sure repository's removal work for legacy and hashed storages
  Use `@hashed` prefix for hashed paths on disk, to avoid collision with existing ones
  Refactor project and storage types
  Prevent using gitlab import task when hashed storage is enabled
  Some codestyle changes and fixes for GitLab pages
  Removed some useless code, codestyle changes and removed an index
  Fix repository reloading in some specs
  Changelog
  Moving away from the "extend" based factory to a more traditional one.
  Enable automatic hashed storage for new projects by application settings
  New storage is now "Hashed" instead of "UUID"
  Add UUID Storage to Project
  Move create_repository back to project model as we can use disk_path and share it
  Codestyle: move hooks to the same place and move dependent methods to private
  Use non-i18n values for setting new group-level issue/MR button text
  indexes external issue tracker
  copyedit
  indexes user/search/ from /user/index
  Correctly encode string params for Gitaly's TreeEntries RPC
  ...

6 files changed, 89 insertions, 0 deletions

diff --git a/db/migrate/20170711145320_add_status_to_ci_stages.rb b/db/migrate/20170711145320_add_status_to_ci_stages.rb
new file mode 100644
index 00000000000..d497a61a959
--- /dev/null
+++ b/db/migrate/20170711145320_add_status_to_ci_stages.rb
@@ -0,0 +1,9 @@
+class AddStatusToCiStages < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_stages, :status, :integer
+  end
+end
diff --git a/db/migrate/20170720111708_add_lock_version_to_ci_stages.rb b/db/migrate/20170720111708_add_lock_version_to_ci_stages.rb
new file mode 100644
index 00000000000..e1c4f033286
--- /dev/null
+++ b/db/migrate/20170720111708_add_lock_version_to_ci_stages.rb
@@ -0,0 +1,9 @@
+class AddLockVersionToCiStages < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_stages, :lock_version, :integer
+  end
+end
diff --git a/db/migrate/20170802013652_add_storage_fields_to_project.rb b/db/migrate/20170802013652_add_storage_fields_to_project.rb
new file mode 100644
index 00000000000..c2381a9d0b2
--- /dev/null
+++ b/db/migrate/20170802013652_add_storage_fields_to_project.rb
@@ -0,0 +1,16 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddStorageFieldsToProject < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    add_column :projects, :storage_version, :integer, limit: 2
+  end
+
+  def down
+    remove_column :projects, :storage_version
+  end
+end
diff --git a/db/migrate/20170807071105_add_hashed_storage_to_settings.rb b/db/migrate/20170807071105_add_hashed_storage_to_settings.rb
new file mode 100644
index 00000000000..0846557add8
--- /dev/null
+++ b/db/migrate/20170807071105_add_hashed_storage_to_settings.rb
@@ -0,0 +1,18 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddHashedStorageToSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :application_settings, :hashed_storage_enabled, :boolean, default: false
+  end
+
+  def down
+    remove_columns :application_settings, :hashed_storage_enabled
+  end
+end
diff --git a/db/post_migrate/20170711145558_migrate_stages_statuses.rb b/db/post_migrate/20170711145558_migrate_stages_statuses.rb
new file mode 100644
index 00000000000..5a24fb1307f
--- /dev/null
+++ b/db/post_migrate/20170711145558_migrate_stages_statuses.rb
@@ -0,0 +1,33 @@
+class MigrateStagesStatuses < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  BATCH_SIZE = 10000
+  RANGE_SIZE = 1000
+  MIGRATION = 'MigrateStageStatus'.freeze
+
+  class Stage < ActiveRecord::Base
+    self.table_name = 'ci_stages'
+    include ::EachBatch
+  end
+
+  def up
+    Stage.where(status: nil).each_batch(of: BATCH_SIZE) do |relation, index|
+      relation.each_batch(of: RANGE_SIZE) do |batch|
+        range = relation.pluck('MIN(id)', 'MAX(id)').first
+        schedule = index * 5.minutes
+
+        BackgroundMigrationWorker.perform_in(schedule, MIGRATION, range)
+      end
+    end
+  end
+
+  def down
+    disable_statement_timeout
+
+    update_column_in_batches(:ci_stages, :status, nil)
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index d22e8235a09..916aa1ffa4c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -128,6 +128,7 @@ ActiveRecord::Schema.define(version: 20170822101017) do
     t.integer "performance_bar_allowed_group_id"
     t.boolean "password_authentication_enabled"
     t.boolean "project_export_enabled", default: true, null: false
+    t.boolean "hashed_storage_enabled", default: false, null: false
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -379,6 +380,8 @@ ActiveRecord::Schema.define(version: 20170822101017) do
     t.datetime "created_at"
     t.datetime "updated_at"
     t.string "name"
+    t.integer "status"
+    t.integer "lock_version"
   end
 
   add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", using: :btree
@@ -1206,6 +1209,7 @@ ActiveRecord::Schema.define(version: 20170822101017) do
     t.datetime "last_repository_updated_at"
     t.string "ci_config_path"
     t.text "delete_error"
+    t.integer "storage_version", limit: 2
   end
 
   add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree