Merge branch 'master' into 'backstage/gb/use-persisted-stages-to-improve-pipelines-table'backstage/gb/use-persisted-stages-to-improve-pipelines-table

 Conflicts:
   app/models/ci/pipeline.rb

11 files changed, 181 insertions, 4 deletions

diff --git a/db/migrate/20160302152808_remove_wrong_import_url_from_projects.rb b/db/migrate/20160302152808_remove_wrong_import_url_from_projects.rb
index 611767ac7fe..95105118764 100644
--- a/db/migrate/20160302152808_remove_wrong_import_url_from_projects.rb
+++ b/db/migrate/20160302152808_remove_wrong_import_url_from_projects.rb
@@ -8,7 +8,7 @@ class RemoveWrongImportUrlFromProjects < ActiveRecord::Migration
     extend AttrEncrypted
     attr_accessor :credentials
     attr_encrypted :credentials,
-                   key: Gitlab::Application.secrets.db_key_base,
+                   key: Settings.attr_encrypted_db_key_base,
                    marshal: true,
                    encode: true,
                    :mode => :per_attribute_iv_and_salt,
diff --git a/db/migrate/20180408143354_rename_users_rss_token_to_feed_token.rb b/db/migrate/20180408143354_rename_users_rss_token_to_feed_token.rb
new file mode 100644
index 00000000000..007cbebaf1b
--- /dev/null
+++ b/db/migrate/20180408143354_rename_users_rss_token_to_feed_token.rb
@@ -0,0 +1,15 @@
+class RenameUsersRssTokenToFeedToken < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    rename_column_concurrently :users, :rss_token, :feed_token
+  end
+
+  def down
+    cleanup_concurrent_column_rename :users, :feed_token, :rss_token
+  end
+end
diff --git a/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb b/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb
new file mode 100644
index 00000000000..e8f0c91d612
--- /dev/null
+++ b/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb
@@ -0,0 +1,13 @@
+class AddPipelineIidToCiPipelines < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    add_column :ci_pipelines, :iid, :integer
+  end
+
+  def down
+    remove_column :ci_pipelines, :iid, :integer
+  end
+end
diff --git a/db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb b/db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb
new file mode 100644
index 00000000000..3fa59b44d5d
--- /dev/null
+++ b/db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb
@@ -0,0 +1,15 @@
+class AddIndexConstraintsToPipelineIid < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :ci_pipelines, [:project_id, :iid], unique: true, where: 'iid IS NOT NULL'
+  end
+
+  def down
+    remove_concurrent_index :ci_pipelines, [:project_id, :iid]
+  end
+end
diff --git a/db/migrate/20180511131058_create_clusters_applications_jupyter.rb b/db/migrate/20180511131058_create_clusters_applications_jupyter.rb
new file mode 100644
index 00000000000..f3923884e37
--- /dev/null
+++ b/db/migrate/20180511131058_create_clusters_applications_jupyter.rb
@@ -0,0 +1,23 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreateClustersApplicationsJupyter < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :clusters_applications_jupyter do |t|
+      t.references :cluster, null: false, unique: true, foreign_key: { on_delete: :cascade }
+      t.references :oauth_application, foreign_key: { on_delete: :nullify }
+
+      t.integer :status, null: false
+      t.string :version, null: false
+      t.string :hostname
+
+      t.timestamps_with_timezone null: false
+
+      t.text :status_reason
+    end
+  end
+end
diff --git a/db/migrate/20180515005612_add_squash_to_merge_requests.rb b/db/migrate/20180515005612_add_squash_to_merge_requests.rb
new file mode 100644
index 00000000000..f526b45bd4b
--- /dev/null
+++ b/db/migrate/20180515005612_add_squash_to_merge_requests.rb
@@ -0,0 +1,19 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddSquashToMergeRequests < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    unless column_exists?(:merge_requests, :squash)
+      add_column_with_default :merge_requests, :squash, :boolean, default: false, allow_null: false
+    end
+  end
+
+  def down
+    remove_column :merge_requests, :squash if column_exists?(:merge_requests, :squash)
+  end
+end
diff --git a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb
index 11b581e4b57..a957f107405 100644
--- a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb
+++ b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb
@@ -48,7 +48,7 @@ class MigrateKubernetesServiceToNewClustersArchitectures < ActiveRecord::Migrati
 
     attr_encrypted :token,
       mode: :per_attribute_iv,
-      key: Gitlab::Application.secrets.db_key_base,
+      key: Settings.attr_encrypted_db_key_base_truncated,
       algorithm: 'aes-256-cbc'
   end
 
diff --git a/db/post_migrate/20180408143355_cleanup_users_rss_token_rename.rb b/db/post_migrate/20180408143355_cleanup_users_rss_token_rename.rb
new file mode 100644
index 00000000000..bff83379087
--- /dev/null
+++ b/db/post_migrate/20180408143355_cleanup_users_rss_token_rename.rb
@@ -0,0 +1,13 @@
+class CleanupUsersRssTokenRename < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    cleanup_concurrent_column_rename :users, :rss_token, :feed_token
+  end
+
+  def down
+    rename_column_concurrently :users, :feed_token, :rss_token
+  end
+end
diff --git a/db/post_migrate/20180507083701_set_minimal_project_build_timeout.rb b/db/post_migrate/20180507083701_set_minimal_project_build_timeout.rb
new file mode 100644
index 00000000000..d9d9e93f5a3
--- /dev/null
+++ b/db/post_migrate/20180507083701_set_minimal_project_build_timeout.rb
@@ -0,0 +1,19 @@
+class SetMinimalProjectBuildTimeout < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MINIMUM_TIMEOUT = 600
+
+  # Allow this migration to resume if it fails partway through
+  disable_ddl_transaction!
+
+  def up
+    update_column_in_batches(:projects, :build_timeout, MINIMUM_TIMEOUT) do |table, query|
+      query.where(table[:build_timeout].lt(MINIMUM_TIMEOUT))
+    end
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/post_migrate/20180521162137_migrate_remaining_mr_metrics_populating_background_migration.rb b/db/post_migrate/20180521162137_migrate_remaining_mr_metrics_populating_background_migration.rb
new file mode 100644
index 00000000000..0282688fa40
--- /dev/null
+++ b/db/post_migrate/20180521162137_migrate_remaining_mr_metrics_populating_background_migration.rb
@@ -0,0 +1,44 @@
+class MigrateRemainingMrMetricsPopulatingBackgroundMigration < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BATCH_SIZE = 5_000
+  MIGRATION = 'PopulateMergeRequestMetricsWithEventsData'
+  DELAY_INTERVAL = 10.minutes
+
+  disable_ddl_transaction!
+
+  class MergeRequest < ActiveRecord::Base
+    self.table_name = 'merge_requests'
+
+    include ::EachBatch
+  end
+
+  def up
+    # Perform any ongoing background migration that might still be running. This
+    # avoids scheduling way too many of the same jobs on self-hosted instances
+    # if they're updating GitLab across multiple versions. The "Take one"
+    # migration was executed on 10.4 on
+    # SchedulePopulateMergeRequestMetricsWithEventsData.
+    Gitlab::BackgroundMigration.steal(MIGRATION)
+
+    metrics_not_exists_clause = <<~SQL
+      NOT EXISTS (SELECT 1 FROM merge_request_metrics
+                  WHERE merge_request_metrics.merge_request_id = merge_requests.id)
+    SQL
+
+    relation = MergeRequest.where(metrics_not_exists_clause)
+
+    # We currently have ~400_000 MR records without metrics on GitLab.com.
+    # This means it'll schedule ~80 jobs (5000 MRs each) with a 10 minutes gap,
+    # so this should take ~14 hours for all background migrations to complete.
+    #
+    queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                          MIGRATION,
+                                                          DELAY_INTERVAL,
+                                                          batch_size: BATCH_SIZE)
+  end
+
+  def down
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index ebb4fa6c533..04d3cce5665 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -451,10 +451,12 @@ ActiveRecord::Schema.define(version: 20180530135500) do
     t.integer "config_source"
     t.boolean "protected"
     t.integer "failure_reason"
+    t.integer "iid"
   end
 
   add_index "ci_pipelines", ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id", using: :btree
   add_index "ci_pipelines", ["pipeline_schedule_id"], name: "index_ci_pipelines_on_pipeline_schedule_id", using: :btree
+  add_index "ci_pipelines", ["project_id", "iid"], name: "index_ci_pipelines_on_project_id_and_iid", unique: true, where: "(iid IS NOT NULL)", using: :btree
   add_index "ci_pipelines", ["project_id", "ref", "status", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_status_and_id", using: :btree
   add_index "ci_pipelines", ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree
   add_index "ci_pipelines", ["project_id"], name: "index_ci_pipelines_on_project_id", using: :btree
@@ -636,6 +638,17 @@ ActiveRecord::Schema.define(version: 20180530135500) do
     t.string "external_ip"
   end
 
+  create_table "clusters_applications_jupyter", force: :cascade do |t|
+    t.integer "cluster_id", null: false
+    t.integer "oauth_application_id"
+    t.integer "status", null: false
+    t.string "version", null: false
+    t.string "hostname"
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.text "status_reason"
+  end
+
   create_table "clusters_applications_prometheus", force: :cascade do |t|
     t.integer "cluster_id", null: false
     t.integer "status", null: false
@@ -1218,6 +1231,7 @@ ActiveRecord::Schema.define(version: 20180530135500) do
     t.integer "latest_merge_request_diff_id"
     t.string "rebase_commit_sha"
     t.boolean "allow_maintainer_to_push"
+    t.boolean "squash", default: false, null: false
   end
 
   add_index "merge_requests", ["assignee_id"], name: "index_merge_requests_on_assignee_id", using: :btree
@@ -2082,9 +2096,9 @@ ActiveRecord::Schema.define(version: 20180530135500) do
     t.date "last_activity_on"
     t.boolean "notified_of_own_activity"
     t.string "preferred_language"
-    t.string "rss_token"
     t.integer "theme_id", limit: 2
     t.integer "accepted_term_id"
+    t.string "feed_token"
   end
 
   add_index "users", ["admin"], name: "index_users_on_admin", using: :btree
@@ -2092,12 +2106,12 @@ ActiveRecord::Schema.define(version: 20180530135500) do
   add_index "users", ["created_at"], name: "index_users_on_created_at", using: :btree
   add_index "users", ["email"], name: "index_users_on_email", unique: true, using: :btree
   add_index "users", ["email"], name: "index_users_on_email_trigram", using: :gin, opclasses: {"email"=>"gin_trgm_ops"}
+  add_index "users", ["feed_token"], name: "index_users_on_feed_token", using: :btree
   add_index "users", ["ghost"], name: "index_users_on_ghost", using: :btree
   add_index "users", ["incoming_email_token"], name: "index_users_on_incoming_email_token", using: :btree
   add_index "users", ["name"], name: "index_users_on_name", using: :btree
   add_index "users", ["name"], name: "index_users_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"}
   add_index "users", ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true, using: :btree
-  add_index "users", ["rss_token"], name: "index_users_on_rss_token", using: :btree
   add_index "users", ["state"], name: "index_users_on_state", using: :btree
   add_index "users", ["username"], name: "index_users_on_username", using: :btree
   add_index "users", ["username"], name: "index_users_on_username_trigram", using: :gin, opclasses: {"username"=>"gin_trgm_ops"}
@@ -2196,6 +2210,8 @@ ActiveRecord::Schema.define(version: 20180530135500) do
   add_foreign_key "clusters", "users", on_delete: :nullify
   add_foreign_key "clusters_applications_helm", "clusters", on_delete: :cascade
   add_foreign_key "clusters_applications_ingress", "clusters", name: "fk_753a7b41c1", on_delete: :cascade
+  add_foreign_key "clusters_applications_jupyter", "clusters", on_delete: :cascade
+  add_foreign_key "clusters_applications_jupyter", "oauth_applications", on_delete: :nullify
   add_foreign_key "clusters_applications_prometheus", "clusters", name: "fk_557e773639", on_delete: :cascade
   add_foreign_key "clusters_applications_runners", "ci_runners", column: "runner_id", name: "fk_02de2ded36", on_delete: :nullify
   add_foreign_key "clusters_applications_runners", "clusters", on_delete: :cascade