Merge commit '8af23def1d6' into object-storage-ee-to-ce-backport

14 files changed, 309 insertions, 1 deletions

diff --git a/db/migrate/20160301174731_add_fingerprint_index.rb b/db/migrate/20160301174731_add_fingerprint_index.rb
new file mode 100644
index 00000000000..f2c3d1ba1ea
--- /dev/null
+++ b/db/migrate/20160301174731_add_fingerprint_index.rb
@@ -0,0 +1,17 @@
+# rubocop:disable all
+class AddFingerprintIndex < ActiveRecord::Migration
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  # https://gitlab.com/gitlab-org/gitlab-ee/issues/764
+  def change
+    args = [:keys, :fingerprint]
+
+    if Gitlab::Database.postgresql?
+      args << { algorithm: :concurrently }
+    end
+
+    add_index(*args) unless index_exists?(:keys, :fingerprint)
+  end
+end
diff --git a/db/migrate/20170425112128_create_pipeline_schedules_table.rb b/db/migrate/20170425112128_create_pipeline_schedules_table.rb
index 57df47f5f42..4f9c56a1ad8 100644
--- a/db/migrate/20170425112128_create_pipeline_schedules_table.rb
+++ b/db/migrate/20170425112128_create_pipeline_schedules_table.rb
@@ -17,7 +17,7 @@ class CreatePipelineSchedulesTable < ActiveRecord::Migration
       t.boolean :active, default: true
       t.datetime :deleted_at
 
-      t.timestamps
+      t.timestamps null: true
     end
 
     add_index(:ci_pipeline_schedules, :project_id)
diff --git a/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb b/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb
new file mode 100644
index 00000000000..1d86a531eb3
--- /dev/null
+++ b/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb
@@ -0,0 +1,19 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddAuthorizedKeysEnabledToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :application_settings, :authorized_keys_enabled, :boolean, default: true, allow_null: false
+  end
+
+  def down
+    remove_column :application_settings, :authorized_keys_enabled
+  end
+end
diff --git a/db/migrate/20170827123848_add_index_on_merge_request_diff_commit_sha.rb b/db/migrate/20170827123848_add_index_on_merge_request_diff_commit_sha.rb
new file mode 100644
index 00000000000..1b360b231a8
--- /dev/null
+++ b/db/migrate/20170827123848_add_index_on_merge_request_diff_commit_sha.rb
@@ -0,0 +1,17 @@
+# rubocop:disable RemoveIndex
+
+class AddIndexOnMergeRequestDiffCommitSha < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :merge_request_diff_commits, :sha, length: Gitlab::Database.mysql? ? 20 : nil
+  end
+
+  def down
+    remove_index :merge_request_diff_commits, :sha if index_exists? :merge_request_diff_commits, :sha
+  end
+end
diff --git a/db/migrate/20170928124105_create_fork_networks.rb b/db/migrate/20170928124105_create_fork_networks.rb
index ca906b953a3..89e5b871967 100644
--- a/db/migrate/20170928124105_create_fork_networks.rb
+++ b/db/migrate/20170928124105_create_fork_networks.rb
@@ -23,6 +23,7 @@ class CreateForkNetworks < ActiveRecord::Migration
     if foreign_keys_for(:fork_networks, :root_project_id).any?
       remove_foreign_key :fork_networks, column: :root_project_id
     end
+
     drop_table :fork_networks
   end
 end
diff --git a/db/migrate/20170928133643_create_fork_network_members.rb b/db/migrate/20170928133643_create_fork_network_members.rb
index 836f023efdc..8c7d9ba859a 100644
--- a/db/migrate/20170928133643_create_fork_network_members.rb
+++ b/db/migrate/20170928133643_create_fork_network_members.rb
@@ -21,6 +21,7 @@ class CreateForkNetworkMembers < ActiveRecord::Migration
     if foreign_keys_for(:fork_network_members, :forked_from_project_id).any?
       remove_foreign_key :fork_network_members, column: :forked_from_project_id
     end
+
     drop_table :fork_network_members
   end
 end
diff --git a/db/migrate/20171207185153_add_merge_request_state_index.rb b/db/migrate/20171207185153_add_merge_request_state_index.rb
new file mode 100644
index 00000000000..72f846c5c38
--- /dev/null
+++ b/db/migrate/20171207185153_add_merge_request_state_index.rb
@@ -0,0 +1,18 @@
+class AddMergeRequestStateIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :merge_requests, [:source_project_id, :source_branch],
+                         where: "state = 'opened'",
+                         name: 'index_merge_requests_on_source_project_and_branch_state_opened'
+  end
+
+  def down
+    remove_concurrent_index_by_name :merge_requests,
+                                    'index_merge_requests_on_source_project_and_branch_state_opened'
+  end
+end
diff --git a/db/migrate/20171211145425_add_can_push_to_deploy_keys_projects.rb b/db/migrate/20171211145425_add_can_push_to_deploy_keys_projects.rb
new file mode 100644
index 00000000000..5dc723db9f9
--- /dev/null
+++ b/db/migrate/20171211145425_add_can_push_to_deploy_keys_projects.rb
@@ -0,0 +1,15 @@
+class AddCanPushToDeployKeysProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :deploy_keys_projects, :can_push, :boolean, default: false, allow_null: false
+  end
+
+  def down
+    remove_column :deploy_keys_projects, :can_push
+  end
+end
diff --git a/db/migrate/20171215113714_populate_can_push_from_deploy_keys_projects.rb b/db/migrate/20171215113714_populate_can_push_from_deploy_keys_projects.rb
new file mode 100644
index 00000000000..680855af945
--- /dev/null
+++ b/db/migrate/20171215113714_populate_can_push_from_deploy_keys_projects.rb
@@ -0,0 +1,64 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class PopulateCanPushFromDeployKeysProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+  DATABASE_NAME = Gitlab::Database.database_name
+
+  disable_ddl_transaction!
+
+  class DeploysKeyProject < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'deploy_keys_projects'
+  end
+
+  def up
+    DeploysKeyProject.each_batch(of: 10_000) do |batch|
+      start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
+
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET deploy_keys_projects.can_push = #{DATABASE_NAME}.keys.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects
+          SET can_push = keys.can_push
+          FROM keys
+          WHERE deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
+    end
+  end
+
+  def down
+    DeploysKeyProject.each_batch(of: 10_000) do |batch|
+      start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
+
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET #{DATABASE_NAME}.keys.can_push = deploy_keys_projects.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE keys
+          SET can_push = deploy_keys_projects.can_push
+          FROM deploy_keys_projects
+          WHERE deploy_keys_projects.deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
+    end
+  end
+end
diff --git a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb
index 7cf1d0cec68..d1a039ed551 100644
--- a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb
+++ b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb
@@ -9,6 +9,7 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration
     return unless Gitlab::Database.postgresql?
 
     disable_statement_timeout
+
     if Gitlab::Database.version.to_f >= 9.5
       # Allow us to hot-patch the index manually ahead of the migration
       execute "CREATE INDEX CONCURRENTLY IF NOT EXISTS #{INDEX_NAME} ON namespaces (lower(name));"
@@ -21,6 +22,7 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration
     return unless Gitlab::Database.postgresql?
 
     disable_statement_timeout
+
     if Gitlab::Database.version.to_f >= 9.2
       execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME};"
     else
diff --git a/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb b/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb
new file mode 100644
index 00000000000..f942b4c062e
--- /dev/null
+++ b/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb
@@ -0,0 +1,29 @@
+class AddCommitsCountToMergeRequestDiff < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  MIGRATION = 'AddMergeRequestDiffCommitsCount'.freeze
+  BATCH_SIZE = 5000
+  DELAY_INTERVAL = 5.minutes.to_i
+
+  class MergeRequestDiff < ActiveRecord::Base
+    self.table_name = 'merge_request_diffs'
+
+    include ::EachBatch
+  end
+
+  disable_ddl_transaction!
+
+  def up
+    add_column :merge_request_diffs, :commits_count, :integer
+
+    say 'Populating the MergeRequestDiff `commits_count`'
+
+    queue_background_migration_jobs_by_range_at_intervals(MergeRequestDiff, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+  def down
+    remove_column :merge_request_diffs, :commits_count
+  end
+end
diff --git a/db/migrate/20180113220114_rework_redirect_routes_indexes.rb b/db/migrate/20180113220114_rework_redirect_routes_indexes.rb
new file mode 100644
index 00000000000..ab9971be074
--- /dev/null
+++ b/db/migrate/20180113220114_rework_redirect_routes_indexes.rb
@@ -0,0 +1,68 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class ReworkRedirectRoutesIndexes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  INDEX_NAME_UNIQUE = "index_redirect_routes_on_path_unique_text_pattern_ops"
+
+  INDEX_NAME_PERM = "index_redirect_routes_on_path_text_pattern_ops_where_permanent"
+  INDEX_NAME_TEMP = "index_redirect_routes_on_path_text_pattern_ops_where_temporary"
+
+  OLD_INDEX_NAME_PATH_TPOPS = "index_redirect_routes_on_path_text_pattern_ops"
+  OLD_INDEX_NAME_PATH_LOWER = "index_on_redirect_routes_lower_path"
+
+  def up
+    disable_statement_timeout
+
+    # this is a plain btree on a single boolean column. It'll never be
+    # selective enough to be valuable. This class is called by
+    # setup_postgresql.rake so it needs to be able to handle this
+    # index not existing.
+    if index_exists?(:redirect_routes, :permanent)
+      remove_concurrent_index(:redirect_routes, :permanent)
+    end
+
+    # If we're on MySQL then the existing index on path is ok. But on
+    # Postgres we need to clean things up:
+    return unless Gitlab::Database.postgresql?
+
+    if_not_exists = Gitlab::Database.version.to_f >= 9.5 ? "IF NOT EXISTS" : ""
+
+    # Unique index on lower(path) across both types of redirect_routes:
+    execute("CREATE UNIQUE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_UNIQUE} ON redirect_routes (lower(path) varchar_pattern_ops);")
+
+    # Make two indexes on path -- one for permanent and one for temporary routes:
+    execute("CREATE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_PERM} ON redirect_routes (lower(path) varchar_pattern_ops) where (permanent);")
+    execute("CREATE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_TEMP} ON redirect_routes (lower(path) varchar_pattern_ops) where (not permanent or permanent is null) ;")
+
+    # Remove the old indexes:
+
+    # This one needed to be on lower(path) but wasn't so it's replaced with the two above
+    execute "DROP INDEX CONCURRENTLY IF EXISTS #{OLD_INDEX_NAME_PATH_TPOPS};"
+
+    # This one isn't needed because we only ever do = and LIKE on this
+    # column so the varchar_pattern_ops index is sufficient
+    execute "DROP INDEX CONCURRENTLY IF EXISTS #{OLD_INDEX_NAME_PATH_LOWER};"
+  end
+
+  def down
+    disable_statement_timeout
+
+    add_concurrent_index(:redirect_routes, :permanent)
+
+    return unless Gitlab::Database.postgresql?
+
+    execute("CREATE INDEX CONCURRENTLY #{OLD_INDEX_NAME_PATH_TPOPS} ON redirect_routes (path varchar_pattern_ops);")
+    execute("CREATE INDEX CONCURRENTLY #{OLD_INDEX_NAME_PATH_LOWER} ON redirect_routes (LOWER(path));")
+
+    execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_UNIQUE};")
+    execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_PERM};")
+    execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_TEMP};")
+  end
+end
diff --git a/db/migrate/20180115201419_add_index_updated_at_to_issues.rb b/db/migrate/20180115201419_add_index_updated_at_to_issues.rb
new file mode 100644
index 00000000000..a5a48fc97be
--- /dev/null
+++ b/db/migrate/20180115201419_add_index_updated_at_to_issues.rb
@@ -0,0 +1,15 @@
+class AddIndexUpdatedAtToIssues < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :issues, :updated_at
+  end
+
+  def down
+    remove_concurrent_index :issues, :updated_at
+  end
+end
diff --git a/db/migrate/20180201145907_migrate_remaining_issues_closed_at.rb b/db/migrate/20180201145907_migrate_remaining_issues_closed_at.rb
new file mode 100644
index 00000000000..7cb913bb2bf
--- /dev/null
+++ b/db/migrate/20180201145907_migrate_remaining_issues_closed_at.rb
@@ -0,0 +1,42 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class MigrateRemainingIssuesClosedAt < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class Issue < ActiveRecord::Base
+    self.table_name = 'issues'
+    include EachBatch
+  end
+
+  def up
+    Gitlab::BackgroundMigration.steal('CopyColumn')
+    Gitlab::BackgroundMigration.steal('CleanupConcurrentTypeChange')
+
+    # It's possible the cleanup job was killed which means we need to manually
+    # migrate any remaining rows.
+    migrate_remaining_rows if migrate_column_type?
+  end
+
+  def down
+  end
+
+  def migrate_remaining_rows
+    Issue.where('closed_at_for_type_change IS NULL AND closed_at IS NOT NULL').each_batch do |batch|
+      batch.update_all('closed_at_for_type_change = closed_at')
+    end
+
+    cleanup_concurrent_column_type_change(:issues, :closed_at)
+  end
+
+  def migrate_column_type?
+    # Some environments may have already executed the previous version of this
+    # migration, thus we don't need to migrate those environments again.
+    column_for('issues', 'closed_at').type == :datetime # rubocop:disable Migration/Datetime
+  end
+end