From d46287cc16ba244720c6d5c00491944336972988 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 20 Sep 2019 12:05:52 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- lib/tasks/gitlab/artifacts/migrate.rake | 29 +++++++++++++------ lib/tasks/gitlab/lfs/migrate.rake | 15 ++++++++++ lib/tasks/gitlab/traces.rake | 38 ------------------------ lib/tasks/gitlab/uploads/migrate.rake | 51 +++++++++------------------------ 4 files changed, 50 insertions(+), 83 deletions(-) delete mode 100644 lib/tasks/gitlab/traces.rake (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake index 9012e55a70c..0d09fd0a4e3 100644 --- a/lib/tasks/gitlab/artifacts/migrate.rake +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -6,18 +6,31 @@ namespace :gitlab do namespace :artifacts do task migrate: :environment do logger = Logger.new(STDOUT) - logger.info('Starting transfer of artifacts') + logger.info('Starting transfer of artifacts to remote storage') - Ci::Build.joins(:project) - .with_artifacts_stored_locally - .find_each(batch_size: 10) do |build| + helper = Gitlab::Artifacts::MigrationHelper.new - build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE) - build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE) + begin + helper.migrate_to_remote_storage do |artifact| + logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to object storage") + end + rescue => e + logger.error(e.message) + end + end + + task migrate_to_local: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of artifacts to local storage') + + helper = Gitlab::Artifacts::MigrationHelper.new - logger.info("Transferred artifact ID #{build.id} with size #{build.artifacts_size} to object storage") + begin + helper.migrate_to_local_storage do |artifact| + logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to local storage") + end rescue => e - logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") + logger.error(e.message) end end end diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake index 97c15175a23..4142903d9c3 100644 --- a/lib/tasks/gitlab/lfs/migrate.rake +++ b/lib/tasks/gitlab/lfs/migrate.rake @@ -17,5 +17,20 @@ namespace :gitlab do logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") end end + + task migrate_to_local: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of LFS files to local storage') + + LfsObject.with_files_stored_remotely + .find_each(batch_size: 10) do |lfs_object| + + lfs_object.file.migrate!(LfsObjectUploader::Store::LOCAL) + + logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to local storage") + rescue => e + logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") + end + end end end diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake deleted file mode 100644 index 5e1ec481ece..00000000000 --- a/lib/tasks/gitlab/traces.rake +++ /dev/null @@ -1,38 +0,0 @@ -require 'logger' -require 'resolv-replace' - -desc "GitLab | Archive legacy traces to trace artifacts" -namespace :gitlab do - namespace :traces do - task archive: :environment do - logger = Logger.new(STDOUT) - logger.info('Archiving legacy traces') - - Ci::Build.finished.without_archived_trace - .order(id: :asc) - .find_in_batches(batch_size: 1000) do |jobs| - job_ids = jobs.map { |job| [job.id] } - - ArchiveTraceWorker.bulk_perform_async(job_ids) - - logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}") - end - end - - task migrate: :environment do - logger = Logger.new(STDOUT) - logger.info('Starting transfer of job traces') - - Ci::Build.joins(:project) - .with_archived_trace_stored_locally - .find_each(batch_size: 10) do |build| - - build.job_artifacts_trace.file.migrate!(ObjectStorage::Store::REMOTE) - - logger.info("Transferred job trace of #{build.id} to object storage") - rescue => e - logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") - end - end - end -end diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index 1c93609a006..44536a447c7 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -3,19 +3,7 @@ namespace :gitlab do namespace :migrate do desc "GitLab | Uploads | Migrate all uploaded files to object storage" task all: :environment do - categories = [%w(AvatarUploader Project :avatar), - %w(AvatarUploader Group :avatar), - %w(AvatarUploader User :avatar), - %w(AttachmentUploader Note :attachment), - %w(AttachmentUploader Appearance :logo), - %w(AttachmentUploader Appearance :header_logo), - %w(FaviconUploader Appearance :favicon), - %w(FileUploader Project), - %w(PersonalFileUploader Snippet), - %w(NamespaceFileUploader Snippet), - %w(FileUploader MergeRequest)] - - categories.each do |args| + Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args| Rake::Task["gitlab:uploads:migrate"].invoke(*args) Rake::Task["gitlab:uploads:migrate"].reenable end @@ -25,34 +13,23 @@ namespace :gitlab do # The following is the actual rake task that migrates uploads of specified # category to object storage desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage' - task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args| - batch_size = ENV.fetch('BATCH', 200).to_i - @to_store = ObjectStorage::Store::REMOTE - @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym - @uploader_class = args.uploader_class.constantize - @model_class = args.model_class.constantize - - uploads.each_batch(of: batch_size, &method(:enqueue_batch)) + task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args| + Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_remote_storage end - def enqueue_batch(batch, index) - job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, - @model_class, - @mounted_as, - @to_store) - puts "Enqueued job ##{index}: #{job}" - rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e - # continue for the next batch - puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red) + namespace :migrate_to_local do + desc "GitLab | Uploads | Migrate all uploaded files to local storage" + task all: :environment do + Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args| + Rake::Task["gitlab:uploads:migrate_to_local"].invoke(*args) + Rake::Task["gitlab:uploads:migrate_to_local"].reenable + end + end end - def uploads - Upload.class_eval { include EachBatch } unless Upload < EachBatch - - Upload - .where(store: [nil, ObjectStorage::Store::LOCAL], - uploader: @uploader_class.to_s, - model_type: @model_class.base_class.sti_name) + desc 'GitLab | Uploads | Migrate the uploaded files of specified type to local storage' + task :migrate_to_local, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args| + Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_local_storage end end end -- cgit v1.2.1