Merge branch 'jej/lfs-object-storage' into 'master'

Can migrate LFS objects to S3 style object storage

Closes #2841

See merge request !2760

3 files changed, 29 insertions, 5 deletions

diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
index 1f4bda6f588..d9436e1d5e5 100644
--- a/lib/backup/artifacts.rb
+++ b/lib/backup/artifacts.rb
@@ -3,7 +3,7 @@ require 'backup/files'
 module Backup
   class Artifacts < Files
     def initialize
-      super('artifacts', ArtifactUploader.local_artifacts_store)
+      super('artifacts', ArtifactUploader.local_store_path)
     end
 
     def create_files_dir
diff --git a/lib/tasks/gitlab/artifacts.rake b/lib/tasks/gitlab/artifacts.rake
index 5676456b2a0..e079177eb3f 100644
--- a/lib/tasks/gitlab/artifacts.rake
+++ b/lib/tasks/gitlab/artifacts.rake
@@ -2,10 +2,12 @@ desc "GitLab | Migrate files for artifacts to comply with new storage format"
 namespace :gitlab do
   namespace :artifacts do
     task migrate: :environment do
-      puts 'Artifacts'.color(:yellow)
-      Ci::Build.joins(:project).with_artifacts
-        .where(artifacts_file_store: ArtifactUploader::LOCAL_STORE)
-        .find_each(batch_size: 100) do |issue|
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of artifacts')
+
+      Ci::Build.joins(:project)
+        .with_artifacts_stored_locally
+        .find_each(batch_size: 10) do |build|
         begin
           build.artifacts_file.migrate!(ArtifactUploader::REMOTE_STORE)
           build.artifacts_metadata.migrate!(ArtifactUploader::REMOTE_STORE)
diff --git a/lib/tasks/gitlab/lfs.rake b/lib/tasks/gitlab/lfs.rake
new file mode 100644
index 00000000000..c17c05f8589
--- /dev/null
+++ b/lib/tasks/gitlab/lfs.rake
@@ -0,0 +1,22 @@
+require 'logger'
+
+desc "GitLab | Migrate LFS objects to remote storage"
+namespace :gitlab do
+  namespace :lfs do
+    task migrate: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of LFS files to object storage')
+
+      LfsObject.with_files_stored_locally
+        .find_each(batch_size: 10) do |lfs_object|
+          begin
+            lfs_object.file.migrate!(LfsObjectUploader::REMOTE_STORE)
+
+            logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage")
+          rescue => e
+            logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
+          end
+        end
+    end
+  end
+end