Merge commit '5b08d59f07fc53c1e34819fac20352119d5343e6' into object-storage-ee-to-ce-backport

17 files changed, 389 insertions, 8 deletions

diff --git a/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb b/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb
index c0cb9d78748..bcdae272209 100644
--- a/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb
+++ b/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb
@@ -16,6 +16,7 @@ class MigrateProcessCommitWorkerJobs < ActiveRecord::Migration
     end
 
     def repository_path
+      # TODO: review if the change from Legacy storage needs to reflect here as well.
       File.join(repository_storage_path, read_attribute(:path_with_namespace) + '.git')
     end
 
diff --git a/db/migrate/20170222111732_create_gpg_keys.rb b/db/migrate/20170222111732_create_gpg_keys.rb
new file mode 100644
index 00000000000..541228e8735
--- /dev/null
+++ b/db/migrate/20170222111732_create_gpg_keys.rb
@@ -0,0 +1,19 @@
+class CreateGpgKeys < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    create_table :gpg_keys do |t|
+      t.timestamps_with_timezone null: false
+
+      t.references :user, index: true, foreign_key: { on_delete: :cascade }
+
+      t.binary :primary_keyid
+      t.binary :fingerprint
+
+      t.text :key
+
+      t.index :primary_keyid, unique: true, length: Gitlab::Database.mysql? ? 20 : nil
+      t.index :fingerprint, unique: true, length: Gitlab::Database.mysql? ? 20 : nil
+    end
+  end
+end
diff --git a/db/migrate/20170428064307_add_column_delete_error_to_projects.rb b/db/migrate/20170428064307_add_column_delete_error_to_projects.rb
new file mode 100644
index 00000000000..09f9d9b5b7a
--- /dev/null
+++ b/db/migrate/20170428064307_add_column_delete_error_to_projects.rb
@@ -0,0 +1,7 @@
+class AddColumnDeleteErrorToProjects < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :projects, :delete_error, :text
+  end
+end
diff --git a/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb b/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb
index 3eaafac321d..af6d10b5158 100644
--- a/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb
+++ b/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb
@@ -62,8 +62,8 @@ class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration
 
     # These columns are not indexed yet, meaning a cascading delete would take
     # forever.
-    add_concurrent_index(:project_group_links, :project_id)
-    add_concurrent_index(:pages_domains, :project_id)
+    add_index_if_not_exists(:project_group_links, :project_id)
+    add_index_if_not_exists(:pages_domains, :project_id)
   end
 
   def down
@@ -71,15 +71,15 @@ class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration
       remove_foreign_key_without_error(source, column)
     end
 
-    add_concurrent_foreign_key(:boards, :projects, column: :project_id)
-    add_concurrent_foreign_key(:lists, :labels, column: :label_id)
-    add_concurrent_foreign_key(:lists, :boards, column: :board_id)
+    add_foreign_key_if_not_exists(:boards, :projects, column: :project_id)
+    add_foreign_key_if_not_exists(:lists, :labels, column: :label_id)
+    add_foreign_key_if_not_exists(:lists, :boards, column: :board_id)
 
-    add_concurrent_foreign_key(:protected_branch_merge_access_levels,
+    add_foreign_key_if_not_exists(:protected_branch_merge_access_levels,
                                :protected_branches,
                                column: :protected_branch_id)
 
-    add_concurrent_foreign_key(:protected_branch_push_access_levels,
+    add_foreign_key_if_not_exists(:protected_branch_push_access_levels,
                                :protected_branches,
                                column: :protected_branch_id)
 
@@ -89,7 +89,7 @@ class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration
 
   def add_foreign_keys
     TABLES.each do |(source, target, column)|
-      add_concurrent_foreign_key(source, target, column: column)
+      add_foreign_key_if_not_exists(source, target, column: column)
     end
   end
 
@@ -153,6 +153,18 @@ class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration
     EOF
   end
 
+  def add_foreign_key_if_not_exists(source, target, column:)
+    return if foreign_key_exists?(source, column)
+
+    add_concurrent_foreign_key(source, target, column: column)
+  end
+
+  def add_index_if_not_exists(table, column)
+    return if index_exists?(table, column)
+
+    add_concurrent_index(table, column)
+  end
+
   def remove_foreign_key_without_error(table, column)
     remove_foreign_key(table, column: column)
   rescue ArgumentError
@@ -163,6 +175,12 @@ class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration
   rescue ArgumentError
   end
 
+  def foreign_key_exists?(table, column)
+    foreign_keys(table).any? do |key|
+      key.options[:column] == column.to_s
+    end
+  end
+
   def connection
     # Rails memoizes connection objects, but this causes them to be shared
     # amongst threads; we don't want that.
diff --git a/db/migrate/20170613154149_create_gpg_signatures.rb b/db/migrate/20170613154149_create_gpg_signatures.rb
new file mode 100644
index 00000000000..f6b5e7ebb7b
--- /dev/null
+++ b/db/migrate/20170613154149_create_gpg_signatures.rb
@@ -0,0 +1,23 @@
+class CreateGpgSignatures < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    create_table :gpg_signatures do |t|
+      t.timestamps_with_timezone null: false
+
+      t.references :project, index: true, foreign_key: { on_delete: :cascade }
+      t.references :gpg_key, index: true, foreign_key: { on_delete: :nullify }
+
+      t.boolean :valid_signature
+
+      t.binary :commit_sha
+      t.binary :gpg_key_primary_keyid
+
+      t.text :gpg_key_user_name
+      t.text :gpg_key_user_email
+
+      t.index :commit_sha, unique: true, length: Gitlab::Database.mysql? ? 20 : nil
+      t.index :gpg_key_primary_keyid, length: Gitlab::Database.mysql? ? 20 : nil
+    end
+  end
+end
diff --git a/db/migrate/20170629171610_rename_application_settings_signin_enabled_to_password_authentication_enabled.rb b/db/migrate/20170629171610_rename_application_settings_signin_enabled_to_password_authentication_enabled.rb
new file mode 100644
index 00000000000..858b3bebace
--- /dev/null
+++ b/db/migrate/20170629171610_rename_application_settings_signin_enabled_to_password_authentication_enabled.rb
@@ -0,0 +1,15 @@
+class RenameApplicationSettingsSigninEnabledToPasswordAuthenticationEnabled < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    rename_column_concurrently :application_settings, :signin_enabled, :password_authentication_enabled
+  end
+
+  def down
+    cleanup_concurrent_column_rename :application_settings, :password_authentication_enabled, :signin_enabled
+  end
+end
diff --git a/db/migrate/20170710083355_clean_stage_id_reference_migration.rb b/db/migrate/20170710083355_clean_stage_id_reference_migration.rb
new file mode 100644
index 00000000000..681203eaf40
--- /dev/null
+++ b/db/migrate/20170710083355_clean_stage_id_reference_migration.rb
@@ -0,0 +1,18 @@
+class CleanStageIdReferenceMigration < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  ##
+  # `MigrateStageIdReferenceInBackground` background migration cleanup.
+  #
+  def up
+    Gitlab::BackgroundMigration.steal('MigrateBuildStageIdReference')
+  end
+
+  def down
+    # noop
+  end
+end
diff --git a/db/migrate/20170713104829_add_foreign_key_to_merge_requests.rb b/db/migrate/20170713104829_add_foreign_key_to_merge_requests.rb
new file mode 100644
index 00000000000..c25d4fd5986
--- /dev/null
+++ b/db/migrate/20170713104829_add_foreign_key_to_merge_requests.rb
@@ -0,0 +1,45 @@
+class AddForeignKeyToMergeRequests < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class MergeRequest < ActiveRecord::Base
+    self.table_name = 'merge_requests'
+    include ::EachBatch
+  end
+
+  def up
+    scope = <<-SQL.strip_heredoc
+      head_pipeline_id IS NOT NULL
+        AND NOT EXISTS (
+          SELECT 1 FROM ci_pipelines
+            WHERE ci_pipelines.id = merge_requests.head_pipeline_id
+        )
+    SQL
+
+    MergeRequest.where(scope).each_batch(of: 1000) do |merge_requests|
+      merge_requests.update_all(head_pipeline_id: nil)
+    end
+
+    unless foreign_key_exists?(:merge_requests, :head_pipeline_id)
+      add_concurrent_foreign_key(:merge_requests, :ci_pipelines,
+                                 column: :head_pipeline_id, on_delete: :nullify)
+    end
+  end
+
+  def down
+    if foreign_key_exists?(:merge_requests, :head_pipeline_id)
+      remove_foreign_key(:merge_requests, column: :head_pipeline_id)
+    end
+  end
+
+  private
+
+  def foreign_key_exists?(table, column)
+    foreign_keys(table).any? do |key|
+      key.options[:column] == column.to_s
+    end
+  end
+end
diff --git a/db/migrate/20170717074009_move_system_upload_folder.rb b/db/migrate/20170717074009_move_system_upload_folder.rb
new file mode 100644
index 00000000000..cce31794115
--- /dev/null
+++ b/db/migrate/20170717074009_move_system_upload_folder.rb
@@ -0,0 +1,60 @@
+class MoveSystemUploadFolder < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    unless file_storage?
+      say 'Using object storage, no need to move.'
+      return
+    end
+
+    unless File.directory?(old_directory)
+      say "#{old_directory} doesn't exist, no need to move it."
+      return
+    end
+
+    FileUtils.mkdir_p(File.join(base_directory, '-'))
+
+    say "Moving #{old_directory} -> #{new_directory}"
+    FileUtils.mv(old_directory, new_directory)
+    FileUtils.ln_s(new_directory, old_directory)
+  end
+
+  def down
+    unless file_storage?
+      say 'Using object storage, no need to move.'
+      return
+    end
+
+    unless File.directory?(new_directory)
+      say "#{new_directory} doesn't exist, no need to move it."
+      return
+    end
+
+    if File.symlink?(old_directory)
+      say "Removing #{old_directory} -> #{new_directory} symlink"
+      FileUtils.rm(old_directory)
+    end
+
+    say "Moving #{new_directory} -> #{old_directory}"
+    FileUtils.mv(new_directory, old_directory)
+  end
+
+  def new_directory
+    File.join(base_directory, '-', 'system')
+  end
+
+  def old_directory
+    File.join(base_directory, 'system')
+  end
+
+  def base_directory
+    File.join(Rails.root, 'public', 'uploads')
+  end
+
+  def file_storage?
+    CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
+  end
+end
diff --git a/db/migrate/20170717200542_add_trusted_column_to_oauth_applications.rb b/db/migrate/20170717200542_add_trusted_column_to_oauth_applications.rb
new file mode 100644
index 00000000000..1a013e6aefb
--- /dev/null
+++ b/db/migrate/20170717200542_add_trusted_column_to_oauth_applications.rb
@@ -0,0 +1,15 @@
+class AddTrustedColumnToOauthApplications < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:oauth_applications, :trusted, :boolean, default: false)
+  end
+
+  def down
+    remove_column(:oauth_applications, :trusted)
+  end
+end
diff --git a/db/migrate/20170720130522_create_ci_pipeline_variables.rb b/db/migrate/20170720130522_create_ci_pipeline_variables.rb
new file mode 100644
index 00000000000..a784f5dd142
--- /dev/null
+++ b/db/migrate/20170720130522_create_ci_pipeline_variables.rb
@@ -0,0 +1,20 @@
+class CreateCiPipelineVariables < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    create_table :ci_pipeline_variables do |t|
+      t.string :key, null: false
+      t.text :value
+      t.text :encrypted_value
+      t.string :encrypted_value_salt
+      t.string :encrypted_value_iv
+      t.integer :pipeline_id, null: false
+    end
+
+    add_index :ci_pipeline_variables, [:pipeline_id, :key], unique: true
+  end
+
+  def down
+    drop_table :ci_pipeline_variables
+  end
+end
diff --git a/db/migrate/20170720130749_add_foreign_key_to_ci_pipeline_variables.rb b/db/migrate/20170720130749_add_foreign_key_to_ci_pipeline_variables.rb
new file mode 100644
index 00000000000..550b8a88f02
--- /dev/null
+++ b/db/migrate/20170720130749_add_foreign_key_to_ci_pipeline_variables.rb
@@ -0,0 +1,15 @@
+class AddForeignKeyToCiPipelineVariables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key(:ci_pipeline_variables, :ci_pipelines, column: :pipeline_id)
+  end
+
+  def down
+    remove_foreign_key(:ci_pipeline_variables, column: :pipeline_id)
+  end
+end
diff --git a/db/migrate/20170724214302_add_lower_path_index_to_redirect_routes.rb b/db/migrate/20170724214302_add_lower_path_index_to_redirect_routes.rb
new file mode 100644
index 00000000000..db60c2087b9
--- /dev/null
+++ b/db/migrate/20170724214302_add_lower_path_index_to_redirect_routes.rb
@@ -0,0 +1,34 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddLowerPathIndexToRedirectRoutes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'index_on_redirect_routes_lower_path'
+
+  disable_ddl_transaction!
+
+  def up
+    return unless Gitlab::Database.postgresql?
+
+    execute "CREATE INDEX CONCURRENTLY #{INDEX_NAME} ON redirect_routes (LOWER(path));"
+  end
+
+  def down
+    return unless Gitlab::Database.postgresql?
+
+    # Why not use remove_concurrent_index_by_name?
+    #
+    # `index_exists?` doesn't work on this index. Perhaps this is related to the
+    # fact that the index doesn't show up in the schema. And apparently it isn't
+    # trivial to write a query that checks for an index. BUT there is a
+    # convenient `IF EXISTS` parameter for `DROP INDEX`.
+    if supports_drop_index_concurrently?
+      disable_statement_timeout
+      execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME};"
+    else
+      execute "DROP INDEX IF EXISTS #{INDEX_NAME};"
+    end
+  end
+end
diff --git a/db/migrate/20170725145659_add_binary_to_merge_request_diff_files.rb b/db/migrate/20170725145659_add_binary_to_merge_request_diff_files.rb
new file mode 100644
index 00000000000..1f5fa7e3d49
--- /dev/null
+++ b/db/migrate/20170725145659_add_binary_to_merge_request_diff_files.rb
@@ -0,0 +1,9 @@
+class AddBinaryToMergeRequestDiffFiles < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :merge_request_diff_files, :binary, :boolean
+  end
+end
diff --git a/db/migrate/20170731175128_add_percentages_to_conv_dev.rb b/db/migrate/20170731175128_add_percentages_to_conv_dev.rb
new file mode 100644
index 00000000000..1819bfc96bb
--- /dev/null
+++ b/db/migrate/20170731175128_add_percentages_to_conv_dev.rb
@@ -0,0 +1,32 @@
+class AddPercentagesToConvDev < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    add_column_with_default :conversational_development_index_metrics, :percentage_boards, :float, allow_null: false, default: 0
+    add_column_with_default :conversational_development_index_metrics, :percentage_ci_pipelines, :float, allow_null: false, default: 0
+    add_column_with_default :conversational_development_index_metrics, :percentage_deployments, :float, allow_null: false, default: 0
+    add_column_with_default :conversational_development_index_metrics, :percentage_environments, :float, allow_null: false, default: 0
+    add_column_with_default :conversational_development_index_metrics, :percentage_issues, :float, allow_null: false, default: 0
+    add_column_with_default :conversational_development_index_metrics, :percentage_merge_requests, :float, allow_null: false, default: 0
+    add_column_with_default :conversational_development_index_metrics, :percentage_milestones, :float, allow_null: false, default: 0
+    add_column_with_default :conversational_development_index_metrics, :percentage_notes, :float, allow_null: false, default: 0
+    add_column_with_default :conversational_development_index_metrics, :percentage_projects_prometheus_active, :float, allow_null: false, default: 0
+    add_column_with_default :conversational_development_index_metrics, :percentage_service_desk_issues, :float, allow_null: false, default: 0
+  end
+
+  def down
+    remove_column :conversational_development_index_metrics, :percentage_boards
+    remove_column :conversational_development_index_metrics, :percentage_ci_pipelines
+    remove_column :conversational_development_index_metrics, :percentage_deployments
+    remove_column :conversational_development_index_metrics, :percentage_environments
+    remove_column :conversational_development_index_metrics, :percentage_issues
+    remove_column :conversational_development_index_metrics, :percentage_merge_requests
+    remove_column :conversational_development_index_metrics, :percentage_milestones
+    remove_column :conversational_development_index_metrics, :percentage_notes
+    remove_column :conversational_development_index_metrics, :percentage_projects_prometheus_active
+    remove_column :conversational_development_index_metrics, :percentage_service_desk_issues
+  end
+end
diff --git a/db/migrate/20170731183033_add_merge_jid_to_merge_requests.rb b/db/migrate/20170731183033_add_merge_jid_to_merge_requests.rb
new file mode 100644
index 00000000000..a7d8f2f3604
--- /dev/null
+++ b/db/migrate/20170731183033_add_merge_jid_to_merge_requests.rb
@@ -0,0 +1,7 @@
+class AddMergeJidToMergeRequests < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :merge_requests, :merge_jid, :string
+  end
+end
diff --git a/db/migrate/20170803130232_reorganise_issues_indexes_for_faster_sorting.rb b/db/migrate/20170803130232_reorganise_issues_indexes_for_faster_sorting.rb
new file mode 100644
index 00000000000..eb7d1be1732
--- /dev/null
+++ b/db/migrate/20170803130232_reorganise_issues_indexes_for_faster_sorting.rb
@@ -0,0 +1,43 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class ReorganiseIssuesIndexesForFasterSorting < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  REMOVE_INDEX_COLUMNS = %i[project_id created_at due_date updated_at].freeze
+
+  ADD_INDEX_COLUMNS = [
+    %i[project_id created_at id state],
+    %i[project_id due_date id state],
+    %i[project_id updated_at id state]
+  ].freeze
+
+  TABLE = :issues
+
+  def up
+    add_indexes(ADD_INDEX_COLUMNS)
+    remove_indexes(REMOVE_INDEX_COLUMNS)
+  end
+
+  def down
+    add_indexes(REMOVE_INDEX_COLUMNS)
+    remove_indexes(ADD_INDEX_COLUMNS)
+  end
+
+  def add_indexes(columns)
+    columns.each do |column|
+      add_concurrent_index(TABLE, column) unless index_exists?(TABLE, column)
+    end
+  end
+
+  def remove_indexes(columns)
+    columns.each do |column|
+      remove_concurrent_index(TABLE, column) if index_exists?(TABLE, column)
+    end
+  end
+end