Merge branch 'master' into live-trace-v2

13 files changed, 275 insertions, 1 deletions

diff --git a/db/migrate/20170301101006_add_ci_runner_namespaces.rb b/db/migrate/20170301101006_add_ci_runner_namespaces.rb
new file mode 100644
index 00000000000..deaf03e928b
--- /dev/null
+++ b/db/migrate/20170301101006_add_ci_runner_namespaces.rb
@@ -0,0 +1,17 @@
+class AddCiRunnerNamespaces < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :ci_runner_namespaces do |t|
+      t.integer :runner_id
+      t.integer :namespace_id
+
+      t.index [:runner_id, :namespace_id], unique: true
+      t.index :namespace_id
+      t.foreign_key :ci_runners, column: :runner_id, on_delete: :cascade
+      t.foreign_key :namespaces, column: :namespace_id, on_delete: :cascade
+    end
+  end
+end
diff --git a/db/migrate/20170906133745_add_runners_token_to_groups.rb b/db/migrate/20170906133745_add_runners_token_to_groups.rb
new file mode 100644
index 00000000000..852f4cba670
--- /dev/null
+++ b/db/migrate/20170906133745_add_runners_token_to_groups.rb
@@ -0,0 +1,9 @@
+class AddRunnersTokenToGroups < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :namespaces, :runners_token, :string
+  end
+end
diff --git a/db/migrate/20180424090541_add_enforce_terms_to_application_settings.rb b/db/migrate/20180424090541_add_enforce_terms_to_application_settings.rb
new file mode 100644
index 00000000000..306cd737771
--- /dev/null
+++ b/db/migrate/20180424090541_add_enforce_terms_to_application_settings.rb
@@ -0,0 +1,9 @@
+class AddEnforceTermsToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :enforce_terms, :boolean, default: false
+  end
+end
diff --git a/db/migrate/20180424134533_create_application_setting_terms.rb b/db/migrate/20180424134533_create_application_setting_terms.rb
new file mode 100644
index 00000000000..f29335cfc51
--- /dev/null
+++ b/db/migrate/20180424134533_create_application_setting_terms.rb
@@ -0,0 +1,13 @@
+class CreateApplicationSettingTerms < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :application_setting_terms do |t|
+      t.integer :cached_markdown_version
+      t.text :terms, null: false
+      t.text :terms_html
+    end
+  end
+end
diff --git a/db/migrate/20180425075446_create_term_agreements.rb b/db/migrate/20180425075446_create_term_agreements.rb
new file mode 100644
index 00000000000..22a9d7b574d
--- /dev/null
+++ b/db/migrate/20180425075446_create_term_agreements.rb
@@ -0,0 +1,28 @@
+class CreateTermAgreements < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    create_table :term_agreements do |t|
+      t.references :term, index: true, null: false
+      t.foreign_key :application_setting_terms, column: :term_id
+      t.references :user, index: true, null: false, foreign_key: { on_delete: :cascade }
+      t.boolean :accepted, default: false, null: false
+
+      t.timestamps_with_timezone null: false
+    end
+
+    add_index :term_agreements, [:user_id, :term_id],
+              unique: true,
+              name: 'term_agreements_unique_index'
+  end
+
+  def down
+    remove_index :term_agreements, name: 'term_agreements_unique_index'
+
+    drop_table :term_agreements
+  end
+end
diff --git a/db/migrate/20180426102016_add_accepted_term_to_users.rb b/db/migrate/20180426102016_add_accepted_term_to_users.rb
new file mode 100644
index 00000000000..3d446f66214
--- /dev/null
+++ b/db/migrate/20180426102016_add_accepted_term_to_users.rb
@@ -0,0 +1,23 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddAcceptedTermToUsers < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    change_table :users do |t|
+      t.references :accepted_term,
+                   null: true
+    end
+    add_concurrent_foreign_key :users, :application_setting_terms, column: :accepted_term_id
+  end
+
+  def down
+    remove_foreign_key :users, column: :accepted_term_id
+    remove_column :users, :accepted_term_id
+  end
+end
diff --git a/db/migrate/20180430101916_add_runner_type_to_ci_runners.rb b/db/migrate/20180430101916_add_runner_type_to_ci_runners.rb
new file mode 100644
index 00000000000..42409349b75
--- /dev/null
+++ b/db/migrate/20180430101916_add_runner_type_to_ci_runners.rb
@@ -0,0 +1,9 @@
+class AddRunnerTypeToCiRunners < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_runners, :runner_type, :smallint
+  end
+end
diff --git a/db/migrate/20180502122856_create_project_mirror_data.rb b/db/migrate/20180502122856_create_project_mirror_data.rb
new file mode 100644
index 00000000000..d449f944844
--- /dev/null
+++ b/db/migrate/20180502122856_create_project_mirror_data.rb
@@ -0,0 +1,20 @@
+class CreateProjectMirrorData < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    return if table_exists?(:project_mirror_data)
+
+    create_table :project_mirror_data do |t|
+      t.references :project, index: true, foreign_key: { on_delete: :cascade }
+      t.string :status
+      t.string :jid
+      t.text :last_error
+    end
+  end
+
+  def down
+    drop_table(:project_mirror_data) if table_exists?(:project_mirror_data)
+  end
+end
diff --git a/db/migrate/20180503150427_add_index_to_namespaces_runners_token.rb b/db/migrate/20180503150427_add_index_to_namespaces_runners_token.rb
new file mode 100644
index 00000000000..4c4e576d49f
--- /dev/null
+++ b/db/migrate/20180503150427_add_index_to_namespaces_runners_token.rb
@@ -0,0 +1,20 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddIndexToNamespacesRunnersToken < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :namespaces, :runners_token, unique: true
+  end
+
+  def down
+    if index_exists?(:namespaces, :runners_token, unique: true)
+      remove_index :namespaces, :runners_token
+    end
+  end
+end
diff --git a/db/migrate/20180503175054_add_indexes_to_project_mirror_data.rb b/db/migrate/20180503175054_add_indexes_to_project_mirror_data.rb
new file mode 100644
index 00000000000..17570269b2e
--- /dev/null
+++ b/db/migrate/20180503175054_add_indexes_to_project_mirror_data.rb
@@ -0,0 +1,17 @@
+class AddIndexesToProjectMirrorData < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :project_mirror_data, :jid
+    add_concurrent_index :project_mirror_data, :status
+  end
+
+  def down
+    remove_index :project_mirror_data, :jid if index_exists? :project_mirror_data, :jid
+    remove_index :project_mirror_data, :status if index_exists? :project_mirror_data, :status
+  end
+end
diff --git a/db/post_migrate/20180430143705_backfill_runner_type_for_ci_runners_post_migrate.rb b/db/post_migrate/20180430143705_backfill_runner_type_for_ci_runners_post_migrate.rb
new file mode 100644
index 00000000000..38af5aae924
--- /dev/null
+++ b/db/post_migrate/20180430143705_backfill_runner_type_for_ci_runners_post_migrate.rb
@@ -0,0 +1,23 @@
+class BackfillRunnerTypeForCiRunnersPostMigrate < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  INSTANCE_RUNNER_TYPE = 1
+  PROJECT_RUNNER_TYPE = 3
+
+  disable_ddl_transaction!
+
+  def up
+    update_column_in_batches(:ci_runners, :runner_type, INSTANCE_RUNNER_TYPE) do |table, query|
+      query.where(table[:is_shared].eq(true)).where(table[:runner_type].eq(nil))
+    end
+
+    update_column_in_batches(:ci_runners, :runner_type, PROJECT_RUNNER_TYPE) do |table, query|
+      query.where(table[:is_shared].eq(false)).where(table[:runner_type].eq(nil))
+    end
+  end
+
+  def down
+  end
+end
diff --git a/db/post_migrate/20180502134117_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb b/db/post_migrate/20180502134117_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb
new file mode 100644
index 00000000000..e39cd33c414
--- /dev/null
+++ b/db/post_migrate/20180502134117_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb
@@ -0,0 +1,38 @@
+class MigrateImportAttributesDataFromProjectsToProjectMirrorData < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  UP_MIGRATION = 'PopulateImportState'.freeze
+  DOWN_MIGRATION = 'RollbackImportStateData'.freeze
+
+  BATCH_SIZE = 1000
+  DELAY_INTERVAL = 5.minutes
+
+  disable_ddl_transaction!
+
+  class Project < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'projects'
+  end
+
+  class ProjectImportState < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'project_mirror_data'
+  end
+
+  def up
+    projects = Project.where.not(import_status: :none)
+
+    queue_background_migration_jobs_by_range_at_intervals(projects, UP_MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+  def down
+    import_state = ProjectImportState.where.not(status: :none)
+
+    queue_background_migration_jobs_by_range_at_intervals(import_state, DOWN_MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+end
diff --git a/db/schema.rb b/db/schema.rb
index 0b4b13c066c..12b752597f9 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180425131009) do
+ActiveRecord::Schema.define(version: 20180503175054) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -40,6 +40,12 @@ ActiveRecord::Schema.define(version: 20180425131009) do
     t.text "new_project_guidelines_html"
   end
 
+  create_table "application_setting_terms", force: :cascade do |t|
+    t.integer "cached_markdown_version"
+    t.text "terms", null: false
+    t.text "terms_html"
+  end
+
   create_table "application_settings", force: :cascade do |t|
     t.integer "default_projects_limit"
     t.boolean "signup_enabled"
@@ -158,6 +164,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
     t.string "auto_devops_domain"
     t.boolean "pages_domain_verification_enabled", default: true, null: false
     t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false
+    t.boolean "enforce_terms", default: false
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -453,6 +460,14 @@ ActiveRecord::Schema.define(version: 20180425131009) do
   add_index "ci_pipelines", ["status"], name: "index_ci_pipelines_on_status", using: :btree
   add_index "ci_pipelines", ["user_id"], name: "index_ci_pipelines_on_user_id", using: :btree
 
+  create_table "ci_runner_namespaces", force: :cascade do |t|
+    t.integer "runner_id"
+    t.integer "namespace_id"
+  end
+
+  add_index "ci_runner_namespaces", ["namespace_id"], name: "index_ci_runner_namespaces_on_namespace_id", using: :btree
+  add_index "ci_runner_namespaces", ["runner_id", "namespace_id"], name: "index_ci_runner_namespaces_on_runner_id_and_namespace_id", unique: true, using: :btree
+
   create_table "ci_runner_projects", force: :cascade do |t|
     t.integer "runner_id", null: false
     t.datetime "created_at"
@@ -481,6 +496,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
     t.integer "access_level", default: 0, null: false
     t.string "ip_address"
     t.integer "maximum_timeout"
+    t.integer "runner_type", limit: 2
   end
 
   add_index "ci_runners", ["contacted_at"], name: "index_ci_runners_on_contacted_at", using: :btree
@@ -1270,6 +1286,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
     t.boolean "require_two_factor_authentication", default: false, null: false
     t.integer "two_factor_grace_period", default: 48, null: false
     t.integer "cached_markdown_version"
+    t.string "runners_token"
   end
 
   add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree
@@ -1280,6 +1297,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
   add_index "namespaces", ["path"], name: "index_namespaces_on_path", using: :btree
   add_index "namespaces", ["path"], name: "index_namespaces_on_path_trigram", using: :gin, opclasses: {"path"=>"gin_trgm_ops"}
   add_index "namespaces", ["require_two_factor_authentication"], name: "index_namespaces_on_require_two_factor_authentication", using: :btree
+  add_index "namespaces", ["runners_token"], name: "index_namespaces_on_runners_token", unique: true, using: :btree
   add_index "namespaces", ["type"], name: "index_namespaces_on_type", using: :btree
 
   create_table "notes", force: :cascade do |t|
@@ -1509,6 +1527,17 @@ ActiveRecord::Schema.define(version: 20180425131009) do
 
   add_index "project_import_data", ["project_id"], name: "index_project_import_data_on_project_id", using: :btree
 
+  create_table "project_mirror_data", force: :cascade do |t|
+    t.integer "project_id"
+    t.string "status"
+    t.string "jid"
+    t.text "last_error"
+  end
+
+  add_index "project_mirror_data", ["jid"], name: "index_project_mirror_data_on_jid", using: :btree
+  add_index "project_mirror_data", ["project_id"], name: "index_project_mirror_data_on_project_id", using: :btree
+  add_index "project_mirror_data", ["status"], name: "index_project_mirror_data_on_status", using: :btree
+
   create_table "project_statistics", force: :cascade do |t|
     t.integer "project_id", null: false
     t.integer "namespace_id", null: false
@@ -1815,6 +1844,18 @@ ActiveRecord::Schema.define(version: 20180425131009) do
 
   add_index "tags", ["name"], name: "index_tags_on_name", unique: true, using: :btree
 
+  create_table "term_agreements", force: :cascade do |t|
+    t.integer "term_id", null: false
+    t.integer "user_id", null: false
+    t.boolean "accepted", default: false, null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+  end
+
+  add_index "term_agreements", ["term_id"], name: "index_term_agreements_on_term_id", using: :btree
+  add_index "term_agreements", ["user_id", "term_id"], name: "term_agreements_unique_index", unique: true, using: :btree
+  add_index "term_agreements", ["user_id"], name: "index_term_agreements_on_user_id", using: :btree
+
   create_table "timelogs", force: :cascade do |t|
     t.integer "time_spent", null: false
     t.integer "user_id"
@@ -2003,6 +2044,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
     t.string "preferred_language"
     t.string "rss_token"
     t.integer "theme_id", limit: 2
+    t.integer "accepted_term_id"
   end
 
   add_index "users", ["admin"], name: "index_users_on_admin", using: :btree
@@ -2097,6 +2139,8 @@ ActiveRecord::Schema.define(version: 20180425131009) do
   add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify
   add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify
   add_foreign_key "ci_pipelines", "projects", name: "fk_86635dbd80", on_delete: :cascade
+  add_foreign_key "ci_runner_namespaces", "ci_runners", column: "runner_id", on_delete: :cascade
+  add_foreign_key "ci_runner_namespaces", "namespaces", on_delete: :cascade
   add_foreign_key "ci_runner_projects", "projects", name: "fk_4478a6f1e4", on_delete: :cascade
   add_foreign_key "ci_stages", "ci_pipelines", column: "pipeline_id", name: "fk_fb57e6cc56", on_delete: :cascade
   add_foreign_key "ci_stages", "projects", name: "fk_2360681d1d", on_delete: :cascade
@@ -2188,6 +2232,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
   add_foreign_key "project_features", "projects", name: "fk_18513d9b92", on_delete: :cascade
   add_foreign_key "project_group_links", "projects", name: "fk_daa8cee94c", on_delete: :cascade
   add_foreign_key "project_import_data", "projects", name: "fk_ffb9ee3a10", on_delete: :cascade
+  add_foreign_key "project_mirror_data", "projects", on_delete: :cascade
   add_foreign_key "project_statistics", "projects", on_delete: :cascade
   add_foreign_key "protected_branch_merge_access_levels", "protected_branches", name: "fk_8a3072ccb3", on_delete: :cascade
   add_foreign_key "protected_branch_push_access_levels", "protected_branches", name: "fk_9ffc86a3d9", on_delete: :cascade
@@ -2202,6 +2247,8 @@ ActiveRecord::Schema.define(version: 20180425131009) do
   add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade
   add_foreign_key "subscriptions", "projects", on_delete: :cascade
   add_foreign_key "system_note_metadata", "notes", name: "fk_d83a918cb1", on_delete: :cascade
+  add_foreign_key "term_agreements", "application_setting_terms", column: "term_id"
+  add_foreign_key "term_agreements", "users", on_delete: :cascade
   add_foreign_key "timelogs", "issues", name: "fk_timelogs_issues_issue_id", on_delete: :cascade
   add_foreign_key "timelogs", "merge_requests", name: "fk_timelogs_merge_requests_merge_request_id", on_delete: :cascade
   add_foreign_key "todos", "notes", name: "fk_91d1f47b13", on_delete: :cascade
@@ -2215,6 +2262,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
   add_foreign_key "user_interacted_projects", "projects", name: "fk_722ceba4f7", on_delete: :cascade
   add_foreign_key "user_interacted_projects", "users", name: "fk_0894651f08", on_delete: :cascade
   add_foreign_key "user_synced_attributes_metadata", "users", on_delete: :cascade
+  add_foreign_key "users", "application_setting_terms", column: "accepted_term_id", name: "fk_789cd90b35", on_delete: :cascade
   add_foreign_key "users_star_projects", "projects", name: "fk_22cd27ddfc", on_delete: :cascade
   add_foreign_key "web_hook_logs", "web_hooks", on_delete: :cascade
   add_foreign_key "web_hooks", "projects", name: "fk_0c8ca6d9d1", on_delete: :cascade