Add background migration to fill file stores

6 files changed, 198 insertions, 0 deletions

diff --git a/db/post_migrate/20180424151928_fill_file_store.rb.rb b/db/post_migrate/20180424151928_fill_file_store.rb.rb
new file mode 100644
index 00000000000..ecf169a4953
--- /dev/null
+++ b/db/post_migrate/20180424151928_fill_file_store.rb.rb
@@ -0,0 +1,95 @@
+class FillFileStore < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class Build < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'ci_builds'
+    BATCH_SIZE = 10_000
+
+    def self.queue_background_migration
+      self.class.where(artifacts_file_store: nil).tap do |relation|
+        queue_background_migration_jobs_by_range_at_intervals(relation,
+          'FillFileStoreBuildArchive',
+          5.minutes,
+          batch_size: BATCH_SIZE)
+      end
+
+      self.class.where(artifacts_metadata_store: nil).tap do |relation|
+        queue_background_migration_jobs_by_range_at_intervals(relation,
+          'FillFileStoreBuildMetadata',
+          5.minutes,
+          batch_size: BATCH_SIZE)
+      end
+    end
+  end
+
+  class JobArtifact < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'ci_job_artifacts'
+    BATCH_SIZE = 10_000
+
+    def self.queue_background_migration
+      self.class.where(file_store: nil).tap do |relation|
+        queue_background_migration_jobs_by_range_at_intervals(relation,
+          'FillFileStoreJobArtifact',
+          5.minutes,
+          batch_size: BATCH_SIZE)
+      end
+    end
+  end
+
+  class LfsObject < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'lfs_objects'
+    BATCH_SIZE = 10_000
+
+    def self.queue_background_migration
+      self.class.where(file_store: nil).tap do |relation|
+        queue_background_migration_jobs_by_range_at_intervals(relation,
+          'FillFileStoreLfsObject',
+          5.minutes,
+          batch_size: BATCH_SIZE)
+      end
+    end
+  end
+
+  class Upload < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'uploads'
+    BATCH_SIZE = 10_000
+
+    def self.queue_background_migration
+      self.class.where(store: nil).tap do |relation|
+        queue_background_migration_jobs_by_range_at_intervals(relation,
+          'FillFileStoreUpload',
+          5.minutes,
+          batch_size: BATCH_SIZE)
+      end
+    end
+  end
+
+  def up
+    disable_statement_timeout
+
+    # Schedule background migrations that fill 'NULL' value by '1' on `file_store`, `store`, `artifacts_file_store` columns
+    # '1' represents ObjectStorage::Store::LOCAL
+    # ci_builds.artifacts_file_store
+    # ci_builds.artifacts_metadata_store
+    # ci_job_artifacts.file_store
+    # lfs_objects.file_store
+    # uploads.store
+
+    FillFileStore::Build.queue_background_migration
+    FillFileStore::JobArtifact.queue_background_migration
+    FillFileStore::LfsObject.queue_background_migration
+    FillFileStore::Upload.queue_background_migration
+  end
+
+  def down
+    # noop
+  end
+end
diff --git a/lib/gitlab/background_migration/fill_file_store_build_archive.rb b/lib/gitlab/background_migration/fill_file_store_build_archive.rb
new file mode 100644
index 00000000000..02d744b7a74
--- /dev/null
+++ b/lib/gitlab/background_migration/fill_file_store_build_archive.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class FillFileStoreBuildArchive
+      class Build < ActiveRecord::Base
+        self.table_name = 'ci_builds'
+        self.inheritance_column = :_type_disabled
+      end
+
+      def perform(start_id, stop_id)
+        FillFileStoreBuildArchive::Build
+          .where('artifacts_file_store = NULL')
+          .where(id: (start_id..stop_id))
+          .update_all(artifacts_file_store: 1)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/fill_file_store_build_metadata.rb b/lib/gitlab/background_migration/fill_file_store_build_metadata.rb
new file mode 100644
index 00000000000..bf1a10b104e
--- /dev/null
+++ b/lib/gitlab/background_migration/fill_file_store_build_metadata.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class FillFileStoreBuildMetadata
+      class Build < ActiveRecord::Base
+        self.table_name = 'ci_builds'
+        self.inheritance_column = :_type_disabled
+      end
+
+      def perform(start_id, stop_id)
+        FillFileStoreBuildMetadata::Build
+          .where('artifacts_metadata_store = NULL')
+          .where(id: (start_id..stop_id))
+          .update_all(artifacts_metadata_store: 1)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb
new file mode 100644
index 00000000000..dfd1e7191d2
--- /dev/null
+++ b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class FillFileStoreJobArtifact
+      class JobArtifact < ActiveRecord::Base
+        self.table_name = 'ci_job_artifacts'
+      end
+
+      def perform(start_id, stop_id)
+        FillFileStoreJobArtifact::JobArtifact
+          .where('file_store = NULL')
+          .where(id: (start_id..stop_id))
+          .update_all(file_store: 1)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb
new file mode 100644
index 00000000000..d423bcfba95
--- /dev/null
+++ b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class FillFileStoreLfsObject
+      class LfsObject < ActiveRecord::Base
+        self.table_name = 'lfs_objects'
+      end
+
+      def perform(start_id, stop_id)
+        FillFileStoreLfsObject::LfsObject
+          .where('file_store = NULL')
+          .where(id: (start_id..stop_id))
+          .update_all(file_store: 1)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/fill_file_store_upload.rb b/lib/gitlab/background_migration/fill_file_store_upload.rb
new file mode 100644
index 00000000000..14f4c667ed8
--- /dev/null
+++ b/lib/gitlab/background_migration/fill_file_store_upload.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class FillFileStoreUpload
+      class Upload < ActiveRecord::Base
+        self.table_name = 'uploads'
+        self.inheritance_column = :_type_disabled
+      end
+
+      def perform(start_id, stop_id)
+        FillFileStoreUpload::Upload
+          .where('store = NULL')
+          .where(id: (start_id..stop_id))
+          .update_all(store: 1)
+      end
+    end
+  end
+end