Add latest changes from gitlab-org/gitlab@master

4 files changed, 88 insertions, 135 deletions

diff --git a/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb b/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb
index be69c10d7c8..afa9ff50146 100644
--- a/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb
+++ b/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb
@@ -11,10 +11,11 @@ describe 'gitlab:artifacts namespace rake task' do
     stub_artifacts_object_storage(enabled: object_storage_enabled)
   end
 
-  subject { run_rake_task('gitlab:artifacts:migrate') }
+  describe 'gitlab:artifacts:migrate' do
+    subject { run_rake_task('gitlab:artifacts:migrate') }
 
-  context 'job artifacts' do
     let!(:artifact) { create(:ci_job_artifact, :archive, file_store: store) }
+    let!(:job_trace) { create(:ci_job_artifact, :trace, file_store: store) }
 
     context 'when local storage is used' do
       let(:store) { ObjectStorage::Store::LOCAL }
@@ -27,6 +28,7 @@ describe 'gitlab:artifacts namespace rake task' do
           subject
 
           expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+          expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
         end
       end
 
@@ -37,6 +39,7 @@ describe 'gitlab:artifacts namespace rake task' do
           subject
 
           expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+          expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
         end
       end
 
@@ -45,6 +48,7 @@ describe 'gitlab:artifacts namespace rake task' do
           subject
 
           expect(artifact.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
+          expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
         end
       end
     end
@@ -57,6 +61,40 @@ describe 'gitlab:artifacts namespace rake task' do
         subject
 
         expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+        expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+      end
+    end
+  end
+
+  describe 'gitlab:artifacts:migrate_to_local' do
+    let(:object_storage_enabled) { true }
+
+    subject { run_rake_task('gitlab:artifacts:migrate_to_local') }
+
+    let!(:artifact) { create(:ci_job_artifact, :archive, file_store: store) }
+    let!(:job_trace) { create(:ci_job_artifact, :trace, file_store: store) }
+
+    context 'when remote storage is used' do
+      let(:store) { ObjectStorage::Store::REMOTE }
+
+      context 'and job has remote file store defined' do
+        it "migrates file to local storage" do
+          subject
+
+          expect(artifact.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
+          expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
+        end
+      end
+    end
+
+    context 'when local storage is used' do
+      let(:store) { ObjectStorage::Store::LOCAL }
+
+      it 'file stays on local storage' do
+        subject
+
+        expect(artifact.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
+        expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
       end
     end
   end
diff --git a/spec/tasks/gitlab/lfs/migrate_rake_spec.rb b/spec/tasks/gitlab/lfs/migrate_rake_spec.rb
index 66d1a192a96..a85a0031a6c 100644
--- a/spec/tasks/gitlab/lfs/migrate_rake_spec.rb
+++ b/spec/tasks/gitlab/lfs/migrate_rake_spec.rb
@@ -5,32 +5,49 @@ describe 'gitlab:lfs namespace rake task' do
     Rake.application.rake_require 'tasks/gitlab/lfs/migrate'
   end
 
-  describe 'migrate' do
+  context 'migration tasks' do
     let(:local) { ObjectStorage::Store::LOCAL }
     let(:remote) { ObjectStorage::Store::REMOTE }
-    let!(:lfs_object) { create(:lfs_object, :with_file, file_store: local) }
 
-    def lfs_migrate
-      run_rake_task('gitlab:lfs:migrate')
+    before do
+      stub_lfs_object_storage(background_upload: false, direct_upload: false)
     end
 
-    context 'object storage disabled' do
-      before do
-        stub_lfs_object_storage(enabled: false)
+    describe 'migrate' do
+      subject { run_rake_task('gitlab:lfs:migrate') }
+
+      let!(:lfs_object) { create(:lfs_object, :with_file) }
+
+      context 'object storage disabled' do
+        before do
+          stub_lfs_object_storage(enabled: false)
+        end
+
+        it "doesn't migrate files" do
+          expect { subject }.not_to change { lfs_object.reload.file_store }
+        end
       end
 
-      it "doesn't migrate files" do
-        expect { lfs_migrate }.not_to change { lfs_object.reload.file_store }
+      context 'object storage enabled' do
+        it 'migrates local file to object storage' do
+          expect { subject }.to change { lfs_object.reload.file_store }.from(local).to(remote)
+        end
       end
     end
 
-    context 'object storage enabled' do
+    describe 'migrate_to_local' do
+      subject { run_rake_task('gitlab:lfs:migrate_to_local') }
+
+      let(:lfs_object) { create(:lfs_object, :with_file, :object_storage) }
+
       before do
-        stub_lfs_object_storage
+        stub_lfs_object_storage(background_upload: false, direct_upload: true)
       end
 
-      it 'migrates local file to object storage' do
-        expect { lfs_migrate }.to change { lfs_object.reload.file_store }.from(local).to(remote)
+      context 'object storage enabled' do
+        it 'migrates remote files to local storage' do
+          expect { subject }.to change { lfs_object.reload.file_store }.from(remote).to(local)
+        end
       end
     end
   end
diff --git a/spec/tasks/gitlab/traces_rake_spec.rb b/spec/tasks/gitlab/traces_rake_spec.rb
deleted file mode 100644
index aaf0d7242dd..00000000000
--- a/spec/tasks/gitlab/traces_rake_spec.rb
+++ /dev/null
@@ -1,113 +0,0 @@
-require 'rake_helper'
-
-describe 'gitlab:traces rake tasks' do
-  before do
-    Rake.application.rake_require 'tasks/gitlab/traces'
-  end
-
-  describe 'gitlab:traces:archive' do
-    shared_examples 'passes the job id to worker' do
-      it do
-        expect(ArchiveTraceWorker).to receive(:bulk_perform_async).with([[job.id]])
-
-        run_rake_task('gitlab:traces:archive')
-      end
-    end
-
-    shared_examples 'does not pass the job id to worker' do
-      it do
-        expect(ArchiveTraceWorker).not_to receive(:bulk_perform_async)
-
-        run_rake_task('gitlab:traces:archive')
-      end
-    end
-
-    context 'when trace file stored in default path' do
-      let!(:job) { create(:ci_build, :success, :trace_live) }
-
-      it_behaves_like 'passes the job id to worker'
-    end
-
-    context 'when trace is stored in database' do
-      let!(:job) { create(:ci_build, :success) }
-
-      before do
-        job.update_column(:trace, 'trace in db')
-      end
-
-      it_behaves_like 'passes the job id to worker'
-    end
-
-    context 'when job has trace artifact' do
-      let!(:job) { create(:ci_build, :success) }
-
-      before do
-        create(:ci_job_artifact, :trace, job: job)
-      end
-
-      it_behaves_like 'does not pass the job id to worker'
-    end
-
-    context 'when job is not finished yet' do
-      let!(:build) { create(:ci_build, :running, :trace_live) }
-
-      it_behaves_like 'does not pass the job id to worker'
-    end
-  end
-
-  describe 'gitlab:traces:migrate' do
-    let(:object_storage_enabled) { false }
-
-    before do
-      stub_artifacts_object_storage(enabled: object_storage_enabled)
-    end
-
-    subject { run_rake_task('gitlab:traces:migrate') }
-
-    let!(:job_trace) { create(:ci_job_artifact, :trace, file_store: store) }
-
-    context 'when local storage is used' do
-      let(:store) { ObjectStorage::Store::LOCAL }
-
-      context 'and job does not have file store defined' do
-        let(:object_storage_enabled) { true }
-        let(:store) { nil }
-
-        it "migrates file to remote storage" do
-          subject
-
-          expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
-        end
-      end
-
-      context 'and remote storage is defined' do
-        let(:object_storage_enabled) { true }
-
-        it "migrates file to remote storage" do
-          subject
-
-          expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
-        end
-      end
-
-      context 'and remote storage is not defined' do
-        it "fails to migrate to remote storage" do
-          subject
-
-          expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
-        end
-      end
-    end
-
-    context 'when remote storage is used' do
-      let(:object_storage_enabled) { true }
-      let(:store) { ObjectStorage::Store::REMOTE }
-
-      it "file stays on remote storage" do
-        subject
-
-        expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
-      end
-    end
-  end
-end
diff --git a/spec/tasks/gitlab/uploads/migrate_rake_spec.rb b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb
index 9588e8be5dc..8d1e355a7d3 100644
--- a/spec/tasks/gitlab/uploads/migrate_rake_spec.rb
+++ b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb
@@ -1,31 +1,42 @@
 require 'rake_helper'
 
-describe 'gitlab:uploads:migrate rake tasks' do
+describe 'gitlab:uploads:migrate and migrate_to_local rake tasks' do
   let(:model_class) { nil }
   let(:uploader_class) { nil }
   let(:mounted_as) { nil }
   let(:batch_size) { 3 }
 
   before do
-    stub_env('BATCH', batch_size.to_s)
+    stub_env('MIGRATION_BATCH_SIZE', batch_size.to_s)
     stub_uploads_object_storage(uploader_class)
     Rake.application.rake_require 'tasks/gitlab/uploads/migrate'
 
     allow(ObjectStorage::MigrateUploadsWorker).to receive(:perform_async)
   end
 
-  def run
+  def run(task)
     args = [uploader_class.to_s, model_class.to_s, mounted_as].compact
-    run_rake_task("gitlab:uploads:migrate", *args)
+    run_rake_task(task, *args)
   end
 
   shared_examples 'enqueue jobs in batch' do |batch:|
-    it do
+    it 'migrates local storage to remote object storage' do
       expect(ObjectStorage::MigrateUploadsWorker)
         .to receive(:perform_async).exactly(batch).times
-              .and_return("A fake job.")
+        .and_return("A fake job.")
 
-      run
+      run('gitlab:uploads:migrate')
+    end
+
+    it 'migrates remote object storage to local storage' do
+      expect(Upload).to receive(:where).exactly(batch + 1).times { Upload.all }
+      expect(ObjectStorage::MigrateUploadsWorker)
+        .to receive(:perform_async)
+        .with(anything, model_class.name, mounted_as, ObjectStorage::Store::LOCAL)
+        .exactly(batch).times
+        .and_return("A fake job.")
+
+      run('gitlab:uploads:migrate_to_local')
     end
   end