diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-09-20 12:05:52 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-09-20 12:05:52 +0000 |
commit | d46287cc16ba244720c6d5c00491944336972988 (patch) | |
tree | bcb8129932d9b734334bfcd67dbcf7b1185d0280 /lib | |
parent | 8ac91ecfd1bb445a0a1572b3c0885c41c9037e8a (diff) | |
download | gitlab-ce-d46287cc16ba244720c6d5c00491944336972988.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/artifacts/migration_helper.rb | 33 | ||||
-rw-r--r-- | lib/gitlab/github_import/importer/releases_importer.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/uploads/migration_helper.rb | 72 | ||||
-rw-r--r-- | lib/tasks/gitlab/artifacts/migrate.rake | 29 | ||||
-rw-r--r-- | lib/tasks/gitlab/lfs/migrate.rake | 15 | ||||
-rw-r--r-- | lib/tasks/gitlab/traces.rake | 38 | ||||
-rw-r--r-- | lib/tasks/gitlab/uploads/migrate.rake | 51 |
7 files changed, 157 insertions, 84 deletions
diff --git a/lib/gitlab/artifacts/migration_helper.rb b/lib/gitlab/artifacts/migration_helper.rb new file mode 100644 index 00000000000..4f047ab3ea8 --- /dev/null +++ b/lib/gitlab/artifacts/migration_helper.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Artifacts + class MigrationHelper + def migrate_to_remote_storage(&block) + artifacts = ::Ci::JobArtifact.with_files_stored_locally + migrate(artifacts, ObjectStorage::Store::REMOTE, &block) + end + + def migrate_to_local_storage(&block) + artifacts = ::Ci::JobArtifact.with_files_stored_remotely + migrate(artifacts, ObjectStorage::Store::LOCAL, &block) + end + + private + + def batch_size + ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i + end + + def migrate(artifacts, store, &block) + artifacts.find_each(batch_size: batch_size) do |artifact| # rubocop:disable CodeReuse/ActiveRecord + artifact.file.migrate!(store) + + yield artifact if block + rescue => e + raise StandardError.new("Failed to transfer artifact of type #{artifact.file_type} and ID #{artifact.id} with error: #{e.message}") + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index 9d925581441..fe3a8d4aea5 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -32,10 +32,11 @@ module Gitlab def build(release) { + name: release.name, tag: release.tag_name, description: description_for(release), created_at: release.created_at, - updated_at: release.updated_at, + updated_at: release.created_at, released_at: release.published_at, project_id: project.id } diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb new file mode 100644 index 00000000000..4ff064007f1 --- /dev/null +++ b/lib/gitlab/uploads/migration_helper.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Uploads + class MigrationHelper + attr_reader :logger + + CATEGORIES = [%w(AvatarUploader Project :avatar), + %w(AvatarUploader Group :avatar), + %w(AvatarUploader User :avatar), + %w(AttachmentUploader Note :attachment), + %w(AttachmentUploader Appearance :logo), + %w(AttachmentUploader Appearance :header_logo), + %w(FaviconUploader Appearance :favicon), + %w(FileUploader Project), + %w(PersonalFileUploader Snippet), + %w(NamespaceFileUploader Snippet), + %w(FileUploader MergeRequest)].freeze + + def initialize(args, logger) + prepare_variables(args, logger) + end + + def migrate_to_remote_storage + @to_store = ObjectStorage::Store::REMOTE + + uploads.each_batch(of: batch_size, &method(:enqueue_batch)) + end + + def migrate_to_local_storage + @to_store = ObjectStorage::Store::LOCAL + + uploads(ObjectStorage::Store::REMOTE).each_batch(of: batch_size, &method(:enqueue_batch)) + end + + private + + def batch_size + ENV.fetch('MIGRATION_BATCH_SIZE', 200).to_i + end + + def prepare_variables(args, logger) + @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym + @uploader_class = args.uploader_class.constantize + @model_class = args.model_class.constantize + @logger = logger + end + + def enqueue_batch(batch, index) + job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, + @model_class, + @mounted_as, + @to_store) + logger.info(message: "[Uploads migration] Enqueued upload migration job", index: index, job_id: job) + rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e + # continue for the next batch + logger.warn(message: "[Uploads migration] Could not enqueue batch", ids: batch.ids, reason: e.message) # rubocop:disable CodeReuse/ActiveRecord + end + + # rubocop:disable CodeReuse/ActiveRecord + def uploads(store_type = [nil, ObjectStorage::Store::LOCAL]) + Upload.class_eval { include EachBatch } unless Upload < EachBatch + + Upload + .where(store: store_type, + uploader: @uploader_class.to_s, + model_type: @model_class.base_class.sti_name) + end + # rubocop:enable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake index 9012e55a70c..0d09fd0a4e3 100644 --- a/lib/tasks/gitlab/artifacts/migrate.rake +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -6,18 +6,31 @@ namespace :gitlab do namespace :artifacts do task migrate: :environment do logger = Logger.new(STDOUT) - logger.info('Starting transfer of artifacts') + logger.info('Starting transfer of artifacts to remote storage') - Ci::Build.joins(:project) - .with_artifacts_stored_locally - .find_each(batch_size: 10) do |build| + helper = Gitlab::Artifacts::MigrationHelper.new - build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE) - build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE) + begin + helper.migrate_to_remote_storage do |artifact| + logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to object storage") + end + rescue => e + logger.error(e.message) + end + end + + task migrate_to_local: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of artifacts to local storage') + + helper = Gitlab::Artifacts::MigrationHelper.new - logger.info("Transferred artifact ID #{build.id} with size #{build.artifacts_size} to object storage") + begin + helper.migrate_to_local_storage do |artifact| + logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to local storage") + end rescue => e - logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") + logger.error(e.message) end end end diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake index 97c15175a23..4142903d9c3 100644 --- a/lib/tasks/gitlab/lfs/migrate.rake +++ b/lib/tasks/gitlab/lfs/migrate.rake @@ -17,5 +17,20 @@ namespace :gitlab do logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") end end + + task migrate_to_local: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of LFS files to local storage') + + LfsObject.with_files_stored_remotely + .find_each(batch_size: 10) do |lfs_object| + + lfs_object.file.migrate!(LfsObjectUploader::Store::LOCAL) + + logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to local storage") + rescue => e + logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") + end + end end end diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake deleted file mode 100644 index 5e1ec481ece..00000000000 --- a/lib/tasks/gitlab/traces.rake +++ /dev/null @@ -1,38 +0,0 @@ -require 'logger' -require 'resolv-replace' - -desc "GitLab | Archive legacy traces to trace artifacts" -namespace :gitlab do - namespace :traces do - task archive: :environment do - logger = Logger.new(STDOUT) - logger.info('Archiving legacy traces') - - Ci::Build.finished.without_archived_trace - .order(id: :asc) - .find_in_batches(batch_size: 1000) do |jobs| - job_ids = jobs.map { |job| [job.id] } - - ArchiveTraceWorker.bulk_perform_async(job_ids) - - logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}") - end - end - - task migrate: :environment do - logger = Logger.new(STDOUT) - logger.info('Starting transfer of job traces') - - Ci::Build.joins(:project) - .with_archived_trace_stored_locally - .find_each(batch_size: 10) do |build| - - build.job_artifacts_trace.file.migrate!(ObjectStorage::Store::REMOTE) - - logger.info("Transferred job trace of #{build.id} to object storage") - rescue => e - logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") - end - end - end -end diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index 1c93609a006..44536a447c7 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -3,19 +3,7 @@ namespace :gitlab do namespace :migrate do desc "GitLab | Uploads | Migrate all uploaded files to object storage" task all: :environment do - categories = [%w(AvatarUploader Project :avatar), - %w(AvatarUploader Group :avatar), - %w(AvatarUploader User :avatar), - %w(AttachmentUploader Note :attachment), - %w(AttachmentUploader Appearance :logo), - %w(AttachmentUploader Appearance :header_logo), - %w(FaviconUploader Appearance :favicon), - %w(FileUploader Project), - %w(PersonalFileUploader Snippet), - %w(NamespaceFileUploader Snippet), - %w(FileUploader MergeRequest)] - - categories.each do |args| + Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args| Rake::Task["gitlab:uploads:migrate"].invoke(*args) Rake::Task["gitlab:uploads:migrate"].reenable end @@ -25,34 +13,23 @@ namespace :gitlab do # The following is the actual rake task that migrates uploads of specified # category to object storage desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage' - task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args| - batch_size = ENV.fetch('BATCH', 200).to_i - @to_store = ObjectStorage::Store::REMOTE - @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym - @uploader_class = args.uploader_class.constantize - @model_class = args.model_class.constantize - - uploads.each_batch(of: batch_size, &method(:enqueue_batch)) + task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args| + Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_remote_storage end - def enqueue_batch(batch, index) - job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, - @model_class, - @mounted_as, - @to_store) - puts "Enqueued job ##{index}: #{job}" - rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e - # continue for the next batch - puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red) + namespace :migrate_to_local do + desc "GitLab | Uploads | Migrate all uploaded files to local storage" + task all: :environment do + Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args| + Rake::Task["gitlab:uploads:migrate_to_local"].invoke(*args) + Rake::Task["gitlab:uploads:migrate_to_local"].reenable + end + end end - def uploads - Upload.class_eval { include EachBatch } unless Upload < EachBatch - - Upload - .where(store: [nil, ObjectStorage::Store::LOCAL], - uploader: @uploader_class.to_s, - model_type: @model_class.base_class.sti_name) + desc 'GitLab | Uploads | Migrate the uploaded files of specified type to local storage' + task :migrate_to_local, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args| + Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_local_storage end end end |