Backports every CE related change from ee-44542 to CE

4 files changed, 88 insertions, 1 deletions

diff --git a/db/migrate/20180502122856_create_project_mirror_data.rb b/db/migrate/20180502122856_create_project_mirror_data.rb
new file mode 100644
index 00000000000..d449f944844
--- /dev/null
+++ b/db/migrate/20180502122856_create_project_mirror_data.rb
@@ -0,0 +1,20 @@
+class CreateProjectMirrorData < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    return if table_exists?(:project_mirror_data)
+
+    create_table :project_mirror_data do |t|
+      t.references :project, index: true, foreign_key: { on_delete: :cascade }
+      t.string :status
+      t.string :jid
+      t.text :last_error
+    end
+  end
+
+  def down
+    drop_table(:project_mirror_data) if table_exists?(:project_mirror_data)
+  end
+end
diff --git a/db/migrate/20180503175054_add_indexes_to_project_mirror_data.rb b/db/migrate/20180503175054_add_indexes_to_project_mirror_data.rb
new file mode 100644
index 00000000000..17570269b2e
--- /dev/null
+++ b/db/migrate/20180503175054_add_indexes_to_project_mirror_data.rb
@@ -0,0 +1,17 @@
+class AddIndexesToProjectMirrorData < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :project_mirror_data, :jid
+    add_concurrent_index :project_mirror_data, :status
+  end
+
+  def down
+    remove_index :project_mirror_data, :jid if index_exists? :project_mirror_data, :jid
+    remove_index :project_mirror_data, :status if index_exists? :project_mirror_data, :status
+  end
+end
diff --git a/db/post_migrate/20180502134117_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb b/db/post_migrate/20180502134117_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb
new file mode 100644
index 00000000000..e39cd33c414
--- /dev/null
+++ b/db/post_migrate/20180502134117_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb
@@ -0,0 +1,38 @@
+class MigrateImportAttributesDataFromProjectsToProjectMirrorData < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  UP_MIGRATION = 'PopulateImportState'.freeze
+  DOWN_MIGRATION = 'RollbackImportStateData'.freeze
+
+  BATCH_SIZE = 1000
+  DELAY_INTERVAL = 5.minutes
+
+  disable_ddl_transaction!
+
+  class Project < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'projects'
+  end
+
+  class ProjectImportState < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'project_mirror_data'
+  end
+
+  def up
+    projects = Project.where.not(import_status: :none)
+
+    queue_background_migration_jobs_by_range_at_intervals(projects, UP_MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+  def down
+    import_state = ProjectImportState.where.not(status: :none)
+
+    queue_background_migration_jobs_by_range_at_intervals(import_state, DOWN_MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+end
diff --git a/db/schema.rb b/db/schema.rb
index 277b14ef7ed..27c70c03612 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180503150427) do
+ActiveRecord::Schema.define(version: 20180503175054) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1518,6 +1518,17 @@ ActiveRecord::Schema.define(version: 20180503150427) do
 
   add_index "project_import_data", ["project_id"], name: "index_project_import_data_on_project_id", using: :btree
 
+  create_table "project_mirror_data", force: :cascade do |t|
+    t.integer "project_id"
+    t.string "status"
+    t.string "jid"
+    t.text "last_error"
+  end
+
+  add_index "project_mirror_data", ["jid"], name: "index_project_mirror_data_on_jid", using: :btree
+  add_index "project_mirror_data", ["project_id"], name: "index_project_mirror_data_on_project_id", using: :btree
+  add_index "project_mirror_data", ["status"], name: "index_project_mirror_data_on_status", using: :btree
+
   create_table "project_statistics", force: :cascade do |t|
     t.integer "project_id", null: false
     t.integer "namespace_id", null: false
@@ -2211,6 +2222,7 @@ ActiveRecord::Schema.define(version: 20180503150427) do
   add_foreign_key "project_features", "projects", name: "fk_18513d9b92", on_delete: :cascade
   add_foreign_key "project_group_links", "projects", name: "fk_daa8cee94c", on_delete: :cascade
   add_foreign_key "project_import_data", "projects", name: "fk_ffb9ee3a10", on_delete: :cascade
+  add_foreign_key "project_mirror_data", "projects", on_delete: :cascade
   add_foreign_key "project_statistics", "projects", on_delete: :cascade
   add_foreign_key "protected_branch_merge_access_levels", "protected_branches", name: "fk_8a3072ccb3", on_delete: :cascade
   add_foreign_key "protected_branch_push_access_levels", "protected_branches", name: "fk_9ffc86a3d9", on_delete: :cascade