From 76f0d7fe6e1a80d5b07eaa792f33d62ec8736d0d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 24 Apr 2018 16:43:19 +0900 Subject: Add background migration to fill file stores --- .../20180424151928_fill_file_store.rb.rb | 95 ++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 db/post_migrate/20180424151928_fill_file_store.rb.rb (limited to 'db') diff --git a/db/post_migrate/20180424151928_fill_file_store.rb.rb b/db/post_migrate/20180424151928_fill_file_store.rb.rb new file mode 100644 index 00000000000..ecf169a4953 --- /dev/null +++ b/db/post_migrate/20180424151928_fill_file_store.rb.rb @@ -0,0 +1,95 @@ +class FillFileStore < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class Build < ActiveRecord::Base + include EachBatch + self.table_name = 'ci_builds' + BATCH_SIZE = 10_000 + + def self.queue_background_migration + self.class.where(artifacts_file_store: nil).tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + 'FillFileStoreBuildArchive', + 5.minutes, + batch_size: BATCH_SIZE) + end + + self.class.where(artifacts_metadata_store: nil).tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + 'FillFileStoreBuildMetadata', + 5.minutes, + batch_size: BATCH_SIZE) + end + end + end + + class JobArtifact < ActiveRecord::Base + include EachBatch + self.table_name = 'ci_job_artifacts' + BATCH_SIZE = 10_000 + + def self.queue_background_migration + self.class.where(file_store: nil).tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + 'FillFileStoreJobArtifact', + 5.minutes, + batch_size: BATCH_SIZE) + end + end + end + + class LfsObject < ActiveRecord::Base + include EachBatch + self.table_name = 'lfs_objects' + BATCH_SIZE = 10_000 + + def self.queue_background_migration + self.class.where(file_store: nil).tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + 'FillFileStoreLfsObject', + 5.minutes, + batch_size: BATCH_SIZE) + end + end + end + + class Upload < ActiveRecord::Base + include EachBatch + self.table_name = 'uploads' + BATCH_SIZE = 10_000 + + def self.queue_background_migration + self.class.where(store: nil).tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + 'FillFileStoreUpload', + 5.minutes, + batch_size: BATCH_SIZE) + end + end + end + + def up + disable_statement_timeout + + # Schedule background migrations that fill 'NULL' value by '1' on `file_store`, `store`, `artifacts_file_store` columns + # '1' represents ObjectStorage::Store::LOCAL + # ci_builds.artifacts_file_store + # ci_builds.artifacts_metadata_store + # ci_job_artifacts.file_store + # lfs_objects.file_store + # uploads.store + + FillFileStore::Build.queue_background_migration + FillFileStore::JobArtifact.queue_background_migration + FillFileStore::LfsObject.queue_background_migration + FillFileStore::Upload.queue_background_migration + end + + def down + # noop + end +end -- cgit v1.2.1 From c0b4f8ac3dea446d2b7bdd04eecf3456f79fe7e7 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 27 Apr 2018 14:31:26 +0900 Subject: Remove legacy artifacts file_store migration --- .../20180424151928_fill_file_store.rb.rb | 36 ++++------------------ 1 file changed, 6 insertions(+), 30 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20180424151928_fill_file_store.rb.rb b/db/post_migrate/20180424151928_fill_file_store.rb.rb index ecf169a4953..d7d53f5814b 100644 --- a/db/post_migrate/20180424151928_fill_file_store.rb.rb +++ b/db/post_migrate/20180424151928_fill_file_store.rb.rb @@ -5,28 +5,6 @@ class FillFileStore < ActiveRecord::Migration disable_ddl_transaction! - class Build < ActiveRecord::Base - include EachBatch - self.table_name = 'ci_builds' - BATCH_SIZE = 10_000 - - def self.queue_background_migration - self.class.where(artifacts_file_store: nil).tap do |relation| - queue_background_migration_jobs_by_range_at_intervals(relation, - 'FillFileStoreBuildArchive', - 5.minutes, - batch_size: BATCH_SIZE) - end - - self.class.where(artifacts_metadata_store: nil).tap do |relation| - queue_background_migration_jobs_by_range_at_intervals(relation, - 'FillFileStoreBuildMetadata', - 5.minutes, - batch_size: BATCH_SIZE) - end - end - end - class JobArtifact < ActiveRecord::Base include EachBatch self.table_name = 'ci_job_artifacts' @@ -75,15 +53,13 @@ class FillFileStore < ActiveRecord::Migration def up disable_statement_timeout - # Schedule background migrations that fill 'NULL' value by '1' on `file_store`, `store`, `artifacts_file_store` columns - # '1' represents ObjectStorage::Store::LOCAL - # ci_builds.artifacts_file_store - # ci_builds.artifacts_metadata_store - # ci_job_artifacts.file_store - # lfs_objects.file_store - # uploads.store + # NOTE: Schedule background migrations that fill 'NULL' value by '1'(ObjectStorage::Store::LOCAL) on `file_store`, `store` columns + # + # Here are the target columns + # - ci_job_artifacts.file_store + # - lfs_objects.file_store + # - uploads.store - FillFileStore::Build.queue_background_migration FillFileStore::JobArtifact.queue_background_migration FillFileStore::LfsObject.queue_background_migration FillFileStore::Upload.queue_background_migration -- cgit v1.2.1 From 4b85b4dc6d22eab375732123fe8a9fba5812000c Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 27 Apr 2018 14:46:03 +0900 Subject: Fix schema and refactoring migration file --- .../20180424151928_fill_file_store.rb.rb | 50 +++++++++++----------- 1 file changed, 26 insertions(+), 24 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20180424151928_fill_file_store.rb.rb b/db/post_migrate/20180424151928_fill_file_store.rb.rb index d7d53f5814b..e950d78e1dd 100644 --- a/db/post_migrate/20180424151928_fill_file_store.rb.rb +++ b/db/post_migrate/20180424151928_fill_file_store.rb.rb @@ -10,13 +10,8 @@ class FillFileStore < ActiveRecord::Migration self.table_name = 'ci_job_artifacts' BATCH_SIZE = 10_000 - def self.queue_background_migration - self.class.where(file_store: nil).tap do |relation| - queue_background_migration_jobs_by_range_at_intervals(relation, - 'FillFileStoreJobArtifact', - 5.minutes, - batch_size: BATCH_SIZE) - end + def self.params_for_background_migration + yield self.where(file_store: nil), 'FillFileStoreJobArtifact', 5.minutes, BATCH_SIZE end end @@ -25,13 +20,8 @@ class FillFileStore < ActiveRecord::Migration self.table_name = 'lfs_objects' BATCH_SIZE = 10_000 - def self.queue_background_migration - self.class.where(file_store: nil).tap do |relation| - queue_background_migration_jobs_by_range_at_intervals(relation, - 'FillFileStoreLfsObject', - 5.minutes, - batch_size: BATCH_SIZE) - end + def self.params_for_background_migration + yield self.where(file_store: nil), 'FillFileStoreLfsObject', 5.minutes, BATCH_SIZE end end @@ -40,13 +30,8 @@ class FillFileStore < ActiveRecord::Migration self.table_name = 'uploads' BATCH_SIZE = 10_000 - def self.queue_background_migration - self.class.where(store: nil).tap do |relation| - queue_background_migration_jobs_by_range_at_intervals(relation, - 'FillFileStoreUpload', - 5.minutes, - batch_size: BATCH_SIZE) - end + def self.params_for_background_migration + yield self.where(store: nil), 'FillFileStoreUpload', 5.minutes, BATCH_SIZE end end @@ -60,9 +45,26 @@ class FillFileStore < ActiveRecord::Migration # - lfs_objects.file_store # - uploads.store - FillFileStore::JobArtifact.queue_background_migration - FillFileStore::LfsObject.queue_background_migration - FillFileStore::Upload.queue_background_migration + FillFileStore::JobArtifact.params_for_background_migration do |relation, class_name, delay_interval, batch_size| + queue_background_migration_jobs_by_range_at_intervals(relation, + class_name, + delay_interval, + batch_size: batch_size) + end + + FillFileStore::LfsObject.params_for_background_migration do |relation, class_name, delay_interval, batch_size| + queue_background_migration_jobs_by_range_at_intervals(relation, + class_name, + delay_interval, + batch_size: batch_size) + end + + FillFileStore::Upload.params_for_background_migration do |relation, class_name, delay_interval, batch_size| + queue_background_migration_jobs_by_range_at_intervals(relation, + class_name, + delay_interval, + batch_size: batch_size) + end end def down -- cgit v1.2.1 From a366fa0988adfe05028395249fc95b967d4fa423 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 27 Apr 2018 15:38:14 +0900 Subject: Add spec for fill_file_store migration --- db/post_migrate/20180424151928_fill_file_store.rb | 73 ++++++++++++++++++++++ .../20180424151928_fill_file_store.rb.rb | 73 ---------------------- 2 files changed, 73 insertions(+), 73 deletions(-) create mode 100644 db/post_migrate/20180424151928_fill_file_store.rb delete mode 100644 db/post_migrate/20180424151928_fill_file_store.rb.rb (limited to 'db') diff --git a/db/post_migrate/20180424151928_fill_file_store.rb b/db/post_migrate/20180424151928_fill_file_store.rb new file mode 100644 index 00000000000..e950d78e1dd --- /dev/null +++ b/db/post_migrate/20180424151928_fill_file_store.rb @@ -0,0 +1,73 @@ +class FillFileStore < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class JobArtifact < ActiveRecord::Base + include EachBatch + self.table_name = 'ci_job_artifacts' + BATCH_SIZE = 10_000 + + def self.params_for_background_migration + yield self.where(file_store: nil), 'FillFileStoreJobArtifact', 5.minutes, BATCH_SIZE + end + end + + class LfsObject < ActiveRecord::Base + include EachBatch + self.table_name = 'lfs_objects' + BATCH_SIZE = 10_000 + + def self.params_for_background_migration + yield self.where(file_store: nil), 'FillFileStoreLfsObject', 5.minutes, BATCH_SIZE + end + end + + class Upload < ActiveRecord::Base + include EachBatch + self.table_name = 'uploads' + BATCH_SIZE = 10_000 + + def self.params_for_background_migration + yield self.where(store: nil), 'FillFileStoreUpload', 5.minutes, BATCH_SIZE + end + end + + def up + disable_statement_timeout + + # NOTE: Schedule background migrations that fill 'NULL' value by '1'(ObjectStorage::Store::LOCAL) on `file_store`, `store` columns + # + # Here are the target columns + # - ci_job_artifacts.file_store + # - lfs_objects.file_store + # - uploads.store + + FillFileStore::JobArtifact.params_for_background_migration do |relation, class_name, delay_interval, batch_size| + queue_background_migration_jobs_by_range_at_intervals(relation, + class_name, + delay_interval, + batch_size: batch_size) + end + + FillFileStore::LfsObject.params_for_background_migration do |relation, class_name, delay_interval, batch_size| + queue_background_migration_jobs_by_range_at_intervals(relation, + class_name, + delay_interval, + batch_size: batch_size) + end + + FillFileStore::Upload.params_for_background_migration do |relation, class_name, delay_interval, batch_size| + queue_background_migration_jobs_by_range_at_intervals(relation, + class_name, + delay_interval, + batch_size: batch_size) + end + end + + def down + # noop + end +end diff --git a/db/post_migrate/20180424151928_fill_file_store.rb.rb b/db/post_migrate/20180424151928_fill_file_store.rb.rb deleted file mode 100644 index e950d78e1dd..00000000000 --- a/db/post_migrate/20180424151928_fill_file_store.rb.rb +++ /dev/null @@ -1,73 +0,0 @@ -class FillFileStore < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - disable_ddl_transaction! - - class JobArtifact < ActiveRecord::Base - include EachBatch - self.table_name = 'ci_job_artifacts' - BATCH_SIZE = 10_000 - - def self.params_for_background_migration - yield self.where(file_store: nil), 'FillFileStoreJobArtifact', 5.minutes, BATCH_SIZE - end - end - - class LfsObject < ActiveRecord::Base - include EachBatch - self.table_name = 'lfs_objects' - BATCH_SIZE = 10_000 - - def self.params_for_background_migration - yield self.where(file_store: nil), 'FillFileStoreLfsObject', 5.minutes, BATCH_SIZE - end - end - - class Upload < ActiveRecord::Base - include EachBatch - self.table_name = 'uploads' - BATCH_SIZE = 10_000 - - def self.params_for_background_migration - yield self.where(store: nil), 'FillFileStoreUpload', 5.minutes, BATCH_SIZE - end - end - - def up - disable_statement_timeout - - # NOTE: Schedule background migrations that fill 'NULL' value by '1'(ObjectStorage::Store::LOCAL) on `file_store`, `store` columns - # - # Here are the target columns - # - ci_job_artifacts.file_store - # - lfs_objects.file_store - # - uploads.store - - FillFileStore::JobArtifact.params_for_background_migration do |relation, class_name, delay_interval, batch_size| - queue_background_migration_jobs_by_range_at_intervals(relation, - class_name, - delay_interval, - batch_size: batch_size) - end - - FillFileStore::LfsObject.params_for_background_migration do |relation, class_name, delay_interval, batch_size| - queue_background_migration_jobs_by_range_at_intervals(relation, - class_name, - delay_interval, - batch_size: batch_size) - end - - FillFileStore::Upload.params_for_background_migration do |relation, class_name, delay_interval, batch_size| - queue_background_migration_jobs_by_range_at_intervals(relation, - class_name, - delay_interval, - batch_size: batch_size) - end - end - - def down - # noop - end -end -- cgit v1.2.1 From 321e580eded961d4de1570d2dd24e8140b0e1b77 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 18 May 2018 17:04:05 +0900 Subject: Rename FillFileStoreUpload to FillStoreUpload --- db/post_migrate/20180424151928_fill_file_store.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'db') diff --git a/db/post_migrate/20180424151928_fill_file_store.rb b/db/post_migrate/20180424151928_fill_file_store.rb index e950d78e1dd..bb7d0978efe 100644 --- a/db/post_migrate/20180424151928_fill_file_store.rb +++ b/db/post_migrate/20180424151928_fill_file_store.rb @@ -28,10 +28,11 @@ class FillFileStore < ActiveRecord::Migration class Upload < ActiveRecord::Base include EachBatch self.table_name = 'uploads' + self.inheritance_column = :_type_disabled # Disable STI BATCH_SIZE = 10_000 def self.params_for_background_migration - yield self.where(store: nil), 'FillFileStoreUpload', 5.minutes, BATCH_SIZE + yield self.where(store: nil), 'FillStoreUpload', 5.minutes, BATCH_SIZE end end -- cgit v1.2.1 From fb77cd72821db2ffb8e3c89fa4bb2b0e95de6658 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 22 May 2018 15:42:52 +0900 Subject: Remove disable_statement_timeout --- db/post_migrate/20180424151928_fill_file_store.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'db') diff --git a/db/post_migrate/20180424151928_fill_file_store.rb b/db/post_migrate/20180424151928_fill_file_store.rb index bb7d0978efe..b41feb233be 100644 --- a/db/post_migrate/20180424151928_fill_file_store.rb +++ b/db/post_migrate/20180424151928_fill_file_store.rb @@ -37,8 +37,6 @@ class FillFileStore < ActiveRecord::Migration end def up - disable_statement_timeout - # NOTE: Schedule background migrations that fill 'NULL' value by '1'(ObjectStorage::Store::LOCAL) on `file_store`, `store` columns # # Here are the target columns -- cgit v1.2.1