Merge branch 'master' into triggermesh-phase1-knative

12 files changed, 322 insertions, 35 deletions

diff --git a/db/migrate/20180910153412_add_token_digest_to_personal_access_tokens.rb b/db/migrate/20180910153412_add_token_digest_to_personal_access_tokens.rb
new file mode 100644
index 00000000000..203fcfe8eae
--- /dev/null
+++ b/db/migrate/20180910153412_add_token_digest_to_personal_access_tokens.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class AddTokenDigestToPersonalAccessTokens < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    change_column :personal_access_tokens, :token, :string, null: true
+
+    add_column :personal_access_tokens, :token_digest, :string
+  end
+
+  def down
+    remove_column :personal_access_tokens, :token_digest
+
+    change_column :personal_access_tokens, :token, :string, null: false
+  end
+end
diff --git a/db/migrate/20180910153413_add_index_to_token_digest_on_personal_access_tokens.rb b/db/migrate/20180910153413_add_index_to_token_digest_on_personal_access_tokens.rb
new file mode 100644
index 00000000000..4300cd13a45
--- /dev/null
+++ b/db/migrate/20180910153413_add_index_to_token_digest_on_personal_access_tokens.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AddIndexToTokenDigestOnPersonalAccessTokens < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :personal_access_tokens, :token_digest, unique: true
+  end
+
+  def down
+    remove_concurrent_index :personal_access_tokens, :token_digest if index_exists?(:personal_access_tokens, :token_digest)
+  end
+end
diff --git a/db/migrate/20181005110927_add_index_to_lfs_objects_file_store.rb b/db/migrate/20181005110927_add_index_to_lfs_objects_file_store.rb
new file mode 100644
index 00000000000..d09543aa4cc
--- /dev/null
+++ b/db/migrate/20181005110927_add_index_to_lfs_objects_file_store.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AddIndexToLfsObjectsFileStore < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :lfs_objects, :file_store
+  end
+
+  def down
+    remove_concurrent_index :lfs_objects, :file_store
+  end
+end
diff --git a/db/migrate/20181010235606_create_board_project_recent_visits.rb b/db/migrate/20181010235606_create_board_project_recent_visits.rb
new file mode 100644
index 00000000000..426f41e202a
--- /dev/null
+++ b/db/migrate/20181010235606_create_board_project_recent_visits.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class CreateBoardProjectRecentVisits < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :board_project_recent_visits, id: :bigserial do |t|
+      t.timestamps_with_timezone null: false
+
+      t.references :user, index: true, foreign_key: { on_delete: :cascade }
+      t.references :project, index: true, foreign_key: { on_delete: :cascade }
+      t.references :board, index: true, foreign_key: { on_delete: :cascade }
+    end
+
+    add_index :board_project_recent_visits, [:user_id, :project_id, :board_id], unique: true, name: 'index_board_project_recent_visits_on_user_project_and_board'
+  end
+end
diff --git a/db/migrate/20181014203236_create_cluster_groups.rb b/db/migrate/20181014203236_create_cluster_groups.rb
new file mode 100644
index 00000000000..69382d5c851
--- /dev/null
+++ b/db/migrate/20181014203236_create_cluster_groups.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class CreateClusterGroups < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :cluster_groups do |t|
+      t.references :cluster, null: false, foreign_key: { on_delete: :cascade }
+      t.references :group, null: false, index: true
+
+      t.index [:cluster_id, :group_id], unique: true
+      t.foreign_key :namespaces, column: :group_id, on_delete: :cascade
+    end
+  end
+end
diff --git a/db/migrate/20181016152238_create_board_group_recent_visits.rb b/db/migrate/20181016152238_create_board_group_recent_visits.rb
new file mode 100644
index 00000000000..1e55dc8658e
--- /dev/null
+++ b/db/migrate/20181016152238_create_board_group_recent_visits.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class CreateBoardGroupRecentVisits < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :board_group_recent_visits, id: :bigserial do |t|
+      t.timestamps_with_timezone null: false
+
+      t.references :user, index: true, foreign_key: { on_delete: :cascade }
+      t.references :board, index: true, foreign_key: { on_delete: :cascade }
+      t.references :group, references: :namespace, column: :group_id, index: true
+      t.foreign_key :namespaces, column: :group_id, on_delete: :cascade
+    end
+
+    add_index :board_group_recent_visits, [:user_id, :group_id, :board_id], unique: true, name: 'index_board_group_recent_visits_on_user_group_and_board'
+  end
+end
diff --git a/db/migrate/20181017001059_add_cluster_type_to_clusters.rb b/db/migrate/20181017001059_add_cluster_type_to_clusters.rb
new file mode 100644
index 00000000000..191e7eb4fb3
--- /dev/null
+++ b/db/migrate/20181017001059_add_cluster_type_to_clusters.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class AddClusterTypeToClusters < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  PROJECT_CLUSTER_TYPE = 3
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:clusters, :cluster_type, :smallint, default: PROJECT_CLUSTER_TYPE)
+  end
+
+  def down
+    remove_column(:clusters, :cluster_type)
+  end
+end
diff --git a/db/migrate/20181031190559_drop_gcp_clusters_table.rb b/db/migrate/20181031190559_drop_gcp_clusters_table.rb
new file mode 100644
index 00000000000..808d474b4fc
--- /dev/null
+++ b/db/migrate/20181031190559_drop_gcp_clusters_table.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class DropGcpClustersTable < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    drop_table :gcp_clusters
+  end
+
+  def down
+    create_table :gcp_clusters do |t|
+      # Order columns by best align scheme
+      t.references :project, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade }
+      t.references :user, foreign_key: { on_delete: :nullify }
+      t.references :service, foreign_key: { on_delete: :nullify }
+      t.integer :status
+      t.integer :gcp_cluster_size, null: false
+
+      # Timestamps
+      t.datetime_with_timezone :created_at, null: false
+      t.datetime_with_timezone :updated_at, null: false
+
+      # Enable/disable
+      t.boolean :enabled, default: true
+
+      # General
+      t.text :status_reason
+
+      # k8s integration specific
+      t.string :project_namespace
+
+      # Cluster details
+      t.string :endpoint
+      t.text :ca_cert
+      t.text :encrypted_kubernetes_token
+      t.string :encrypted_kubernetes_token_iv
+      t.string :username
+      t.text :encrypted_password
+      t.string :encrypted_password_iv
+
+      # GKE
+      t.string :gcp_project_id, null: false
+      t.string :gcp_cluster_zone, null: false
+      t.string :gcp_cluster_name, null: false
+      t.string :gcp_machine_type
+      t.string :gcp_operation_id
+      t.text :encrypted_gcp_token
+      t.string :encrypted_gcp_token_iv
+    end
+  end
+end
diff --git a/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb b/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb
index c48f1c938d0..3ae4406ff96 100644
--- a/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb
+++ b/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb
@@ -13,7 +13,7 @@ class CleanupMoveSystemUploadFolderSymlink < ActiveRecord::Migration
       say "Removing #{old_directory} -> #{new_directory} symlink"
       FileUtils.rm(old_directory)
     else
-      say "Symlink #{old_directory} non existant, nothing to do."
+      say "Symlink #{old_directory} non existent, nothing to do."
     end
   end
 
diff --git a/db/post_migrate/20180913142237_schedule_digest_personal_access_tokens.rb b/db/post_migrate/20180913142237_schedule_digest_personal_access_tokens.rb
new file mode 100644
index 00000000000..36be819b245
--- /dev/null
+++ b/db/post_migrate/20180913142237_schedule_digest_personal_access_tokens.rb
@@ -0,0 +1,28 @@
+class ScheduleDigestPersonalAccessTokens < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  BATCH_SIZE = 10_000
+  MIGRATION = 'DigestColumn'
+  DELAY_INTERVAL = 5.minutes.to_i
+
+  disable_ddl_transaction!
+
+  class PersonalAccessToken < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'personal_access_tokens'
+  end
+
+  def up
+    PersonalAccessToken.where('token is NOT NULL').each_batch(of: BATCH_SIZE) do |batch, index|
+      range = batch.pluck('MIN(id)', 'MAX(id)').first
+      BackgroundMigrationWorker.perform_in(index * DELAY_INTERVAL, MIGRATION, ['PersonalAccessToken', :token, :token_digest, *range])
+    end
+  end
+
+  def down
+    # raise ActiveRecord::IrreversibleMigration
+  end
+end
diff --git a/db/post_migrate/20181014121030_enqueue_redact_links.rb b/db/post_migrate/20181014121030_enqueue_redact_links.rb
new file mode 100644
index 00000000000..1ee4703c88a
--- /dev/null
+++ b/db/post_migrate/20181014121030_enqueue_redact_links.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+class EnqueueRedactLinks < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BATCH_SIZE = 1000
+  DELAY_INTERVAL = 5.minutes.to_i
+  MIGRATION = 'RedactLinks'
+
+  disable_ddl_transaction!
+
+  class Note < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'notes'
+    self.inheritance_column = :_type_disabled
+  end
+
+  class Issue < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'issues'
+    self.inheritance_column = :_type_disabled
+  end
+
+  class MergeRequest < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'merge_requests'
+    self.inheritance_column = :_type_disabled
+  end
+
+  class Snippet < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'snippets'
+    self.inheritance_column = :_type_disabled
+  end
+
+  def up
+    disable_statement_timeout do
+      schedule_migration(Note, 'note')
+      schedule_migration(Issue, 'description')
+      schedule_migration(MergeRequest, 'description')
+      schedule_migration(Snippet, 'description')
+    end
+  end
+
+  def down
+    # nothing to do
+  end
+
+  private
+
+  def schedule_migration(model, field)
+    link_pattern = "%/sent_notifications/" + ("_" * 32) + "/unsubscribe%"
+
+    model.where("#{field} like ?", link_pattern).each_batch(of: BATCH_SIZE) do |batch, index|
+      start_id, stop_id = batch.pluck('MIN(id)', 'MAX(id)').first
+
+      BackgroundMigrationWorker.perform_in(index * DELAY_INTERVAL, MIGRATION, [model.name.demodulize, field, start_id, stop_id])
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 7ef3d1941b2..4ed585e7110 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20181013005024) do
+ActiveRecord::Schema.define(version: 20181031190559) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -204,6 +204,32 @@ ActiveRecord::Schema.define(version: 20181013005024) do
   add_index "badges", ["group_id"], name: "index_badges_on_group_id", using: :btree
   add_index "badges", ["project_id"], name: "index_badges_on_project_id", using: :btree
 
+  create_table "board_group_recent_visits", id: :bigserial, force: :cascade do |t|
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.integer "user_id"
+    t.integer "board_id"
+    t.integer "group_id"
+  end
+
+  add_index "board_group_recent_visits", ["board_id"], name: "index_board_group_recent_visits_on_board_id", using: :btree
+  add_index "board_group_recent_visits", ["group_id"], name: "index_board_group_recent_visits_on_group_id", using: :btree
+  add_index "board_group_recent_visits", ["user_id", "group_id", "board_id"], name: "index_board_group_recent_visits_on_user_group_and_board", unique: true, using: :btree
+  add_index "board_group_recent_visits", ["user_id"], name: "index_board_group_recent_visits_on_user_id", using: :btree
+
+  create_table "board_project_recent_visits", id: :bigserial, force: :cascade do |t|
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.integer "user_id"
+    t.integer "project_id"
+    t.integer "board_id"
+  end
+
+  add_index "board_project_recent_visits", ["board_id"], name: "index_board_project_recent_visits_on_board_id", using: :btree
+  add_index "board_project_recent_visits", ["project_id"], name: "index_board_project_recent_visits_on_project_id", using: :btree
+  add_index "board_project_recent_visits", ["user_id", "project_id", "board_id"], name: "index_board_project_recent_visits_on_user_project_and_board", unique: true, using: :btree
+  add_index "board_project_recent_visits", ["user_id"], name: "index_board_project_recent_visits_on_user_id", using: :btree
+
   create_table "boards", force: :cascade do |t|
     t.integer "project_id"
     t.datetime "created_at", null: false
@@ -573,6 +599,14 @@ ActiveRecord::Schema.define(version: 20181013005024) do
 
   add_index "ci_variables", ["project_id", "key", "environment_scope"], name: "index_ci_variables_on_project_id_and_key_and_environment_scope", unique: true, using: :btree
 
+  create_table "cluster_groups", force: :cascade do |t|
+    t.integer "cluster_id", null: false
+    t.integer "group_id", null: false
+  end
+
+  add_index "cluster_groups", ["cluster_id", "group_id"], name: "index_cluster_groups_on_cluster_id_and_group_id", unique: true, using: :btree
+  add_index "cluster_groups", ["group_id"], name: "index_cluster_groups_on_group_id", using: :btree
+
   create_table "cluster_platforms_kubernetes", force: :cascade do |t|
     t.integer "cluster_id", null: false
     t.datetime_with_timezone "created_at", null: false
@@ -628,6 +662,7 @@ ActiveRecord::Schema.define(version: 20181013005024) do
     t.boolean "enabled", default: true
     t.string "name", null: false
     t.string "environment_scope", default: "*", null: false
+    t.integer "cluster_type", limit: 2, default: 3, null: false
   end
 
   add_index "clusters", ["enabled"], name: "index_clusters_on_enabled", using: :btree
@@ -893,35 +928,6 @@ ActiveRecord::Schema.define(version: 20181013005024) do
 
   add_index "forked_project_links", ["forked_to_project_id"], name: "index_forked_project_links_on_forked_to_project_id", unique: true, using: :btree
 
-  create_table "gcp_clusters", force: :cascade do |t|
-    t.integer "project_id", null: false
-    t.integer "user_id"
-    t.integer "service_id"
-    t.integer "status"
-    t.integer "gcp_cluster_size", null: false
-    t.datetime_with_timezone "created_at", null: false
-    t.datetime_with_timezone "updated_at", null: false
-    t.boolean "enabled", default: true
-    t.text "status_reason"
-    t.string "project_namespace"
-    t.string "endpoint"
-    t.text "ca_cert"
-    t.text "encrypted_kubernetes_token"
-    t.string "encrypted_kubernetes_token_iv"
-    t.string "username"
-    t.text "encrypted_password"
-    t.string "encrypted_password_iv"
-    t.string "gcp_project_id", null: false
-    t.string "gcp_cluster_zone", null: false
-    t.string "gcp_cluster_name", null: false
-    t.string "gcp_machine_type"
-    t.string "gcp_operation_id"
-    t.text "encrypted_gcp_token"
-    t.string "encrypted_gcp_token_iv"
-  end
-
-  add_index "gcp_clusters", ["project_id"], name: "index_gcp_clusters_on_project_id", unique: true, using: :btree
-
   create_table "gpg_key_subkeys", force: :cascade do |t|
     t.integer "gpg_key_id", null: false
     t.binary "keyid"
@@ -1142,6 +1148,7 @@ ActiveRecord::Schema.define(version: 20181013005024) do
     t.integer "file_store"
   end
 
+  add_index "lfs_objects", ["file_store"], name: "index_lfs_objects_on_file_store", using: :btree
   add_index "lfs_objects", ["oid"], name: "index_lfs_objects_on_oid", unique: true, using: :btree
 
   create_table "lfs_objects_projects", force: :cascade do |t|
@@ -1523,7 +1530,7 @@ ActiveRecord::Schema.define(version: 20181013005024) do
 
   create_table "personal_access_tokens", force: :cascade do |t|
     t.integer "user_id", null: false
-    t.string "token", null: false
+    t.string "token"
     t.string "name", null: false
     t.boolean "revoked", default: false
     t.date "expires_at"
@@ -1531,9 +1538,11 @@ ActiveRecord::Schema.define(version: 20181013005024) do
     t.datetime "updated_at", null: false
     t.string "scopes", default: "--- []\n", null: false
     t.boolean "impersonation", default: false, null: false
+    t.string "token_digest"
   end
 
   add_index "personal_access_tokens", ["token"], name: "index_personal_access_tokens_on_token", unique: true, using: :btree
+  add_index "personal_access_tokens", ["token_digest"], name: "index_personal_access_tokens_on_token_digest", unique: true, using: :btree
   add_index "personal_access_tokens", ["user_id"], name: "index_personal_access_tokens_on_user_id", using: :btree
 
   create_table "programming_languages", force: :cascade do |t|
@@ -2316,6 +2325,12 @@ ActiveRecord::Schema.define(version: 20181013005024) do
   add_foreign_key "application_settings", "users", column: "usage_stats_set_by_user_id", name: "fk_964370041d", on_delete: :nullify
   add_foreign_key "badges", "namespaces", column: "group_id", on_delete: :cascade
   add_foreign_key "badges", "projects", on_delete: :cascade
+  add_foreign_key "board_group_recent_visits", "boards", on_delete: :cascade
+  add_foreign_key "board_group_recent_visits", "namespaces", column: "group_id", on_delete: :cascade
+  add_foreign_key "board_group_recent_visits", "users", on_delete: :cascade
+  add_foreign_key "board_project_recent_visits", "boards", on_delete: :cascade
+  add_foreign_key "board_project_recent_visits", "projects", on_delete: :cascade
+  add_foreign_key "board_project_recent_visits", "users", on_delete: :cascade
   add_foreign_key "boards", "namespaces", column: "group_id", on_delete: :cascade
   add_foreign_key "boards", "projects", name: "fk_f15266b5f9", on_delete: :cascade
   add_foreign_key "chat_teams", "namespaces", on_delete: :cascade
@@ -2350,6 +2365,8 @@ ActiveRecord::Schema.define(version: 20181013005024) do
   add_foreign_key "ci_triggers", "projects", name: "fk_e3e63f966e", on_delete: :cascade
   add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade
   add_foreign_key "ci_variables", "projects", name: "fk_ada5eb64b3", on_delete: :cascade
+  add_foreign_key "cluster_groups", "clusters", on_delete: :cascade
+  add_foreign_key "cluster_groups", "namespaces", column: "group_id", on_delete: :cascade
   add_foreign_key "cluster_platforms_kubernetes", "clusters", on_delete: :cascade
   add_foreign_key "cluster_projects", "clusters", on_delete: :cascade
   add_foreign_key "cluster_projects", "projects", on_delete: :cascade
@@ -2377,9 +2394,6 @@ ActiveRecord::Schema.define(version: 20181013005024) do
   add_foreign_key "fork_network_members", "projects", on_delete: :cascade
   add_foreign_key "fork_networks", "projects", column: "root_project_id", name: "fk_e7b436b2b5", on_delete: :nullify
   add_foreign_key "forked_project_links", "projects", column: "forked_to_project_id", name: "fk_434510edb0", on_delete: :cascade
-  add_foreign_key "gcp_clusters", "projects", on_delete: :cascade
-  add_foreign_key "gcp_clusters", "services", on_delete: :nullify
-  add_foreign_key "gcp_clusters", "users", on_delete: :nullify
   add_foreign_key "gpg_key_subkeys", "gpg_keys", on_delete: :cascade
   add_foreign_key "gpg_keys", "users", on_delete: :cascade
   add_foreign_key "gpg_signatures", "gpg_key_subkeys", on_delete: :nullify