Use FastDestroy for deleting uploads

It gathers list of file paths to delete before destroying
the parent object. Then after the parent_object is destroyed
these paths are scheduled for deletion asynchronously.

Carrierwave needed associated model for deleting upload file.
To avoid this requirement, simple Fog/File layer is used directly
for file deletion, this allows us to use just a simple list of paths.

8 files changed, 152 insertions, 11 deletions

diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml
index 7df129da95a..bae5b21c26f 100644
--- a/spec/lib/gitlab/import_export/all_models.yml
+++ b/spec/lib/gitlab/import_export/all_models.yml
@@ -287,6 +287,7 @@ project:
 - statistics
 - container_repositories
 - uploads
+- file_uploads
 - import_state
 - members_and_requesters
 - build_trace_section_names
diff --git a/spec/models/appearance_spec.rb b/spec/models/appearance_spec.rb
index 77b07cf1ac9..35415030154 100644
--- a/spec/models/appearance_spec.rb
+++ b/spec/models/appearance_spec.rb
@@ -20,7 +20,7 @@ describe Appearance do
   end
 
   context 'with uploads' do
-    it_behaves_like 'model with mounted uploader', false do
+    it_behaves_like 'model with uploads', false do
       let(:model_object) { create(:appearance, :with_logo) }
       let(:upload_attribute) { :logo }
       let(:uploader_class) { AttachmentUploader }
diff --git a/spec/models/group_spec.rb b/spec/models/group_spec.rb
index 87aa5a46c21..e63881242f6 100644
--- a/spec/models/group_spec.rb
+++ b/spec/models/group_spec.rb
@@ -739,7 +739,7 @@ describe Group do
   end
 
   context 'with uploads' do
-    it_behaves_like 'model with mounted uploader', true do
+    it_behaves_like 'model with uploads', true do
       let(:model_object) { create(:group, :with_avatar) }
       let(:upload_attribute) { :avatar }
       let(:uploader_class) { AttachmentUploader }
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index 50920d9d1fc..93c83fd21fd 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -3898,7 +3898,7 @@ describe Project do
   end
 
   context 'with uploads' do
-    it_behaves_like 'model with mounted uploader', true do
+    it_behaves_like 'model with uploads', true do
       let(:model_object) { create(:project, :with_avatar) }
       let(:upload_attribute) { :avatar }
       let(:uploader_class) { AttachmentUploader }
diff --git a/spec/models/uploads/fog_spec.rb b/spec/models/uploads/fog_spec.rb
new file mode 100644
index 00000000000..4a44cf5ab0f
--- /dev/null
+++ b/spec/models/uploads/fog_spec.rb
@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Uploads::Fog do
+  let(:data_store) { described_class.new }
+
+  before do
+    stub_uploads_object_storage(FileUploader)
+  end
+
+  describe '#available?' do
+    subject { data_store.available? }
+
+    context 'when object storage is enabled' do
+      it { is_expected.to be_truthy }
+    end
+
+    context 'when object storage is disabled' do
+      before do
+        stub_uploads_object_storage(FileUploader, enabled: false)
+      end
+
+      it { is_expected.to be_falsy }
+    end
+  end
+
+  context 'model with uploads' do
+    let(:project) { create(:project) }
+    let(:relation) { project.uploads }
+
+    describe '#keys' do
+      let!(:uploads) { create_list(:upload, 2, :object_storage, uploader: FileUploader, model: project) }
+      subject { data_store.keys(relation) }
+
+      it 'returns keys' do
+        is_expected.to match_array(relation.pluck(:path))
+      end
+    end
+
+    describe '#delete_keys' do
+      let(:keys) { data_store.keys(relation) }
+      let!(:uploads) { create_list(:upload, 2, :with_file, :issuable_upload, model: project) }
+      subject { data_store.delete_keys(keys) }
+
+      before do
+        uploads.each { |upload| upload.build_uploader.migrate!(2) }
+      end
+
+      it 'deletes multiple data' do
+        paths = relation.pluck(:path)
+
+        ::Fog::Storage.new(FileUploader.object_store_credentials).tap do |connection|
+          paths.each do |path|
+            expect(connection.get_object('uploads', path)[:body]).not_to be_nil
+          end
+        end
+
+        subject
+
+        ::Fog::Storage.new(FileUploader.object_store_credentials).tap do |connection|
+          paths.each do |path|
+            expect { connection.get_object('uploads', path)[:body] }.to raise_error(Excon::Error::NotFound)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/spec/models/uploads/local_spec.rb b/spec/models/uploads/local_spec.rb
new file mode 100644
index 00000000000..3468399f370
--- /dev/null
+++ b/spec/models/uploads/local_spec.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Uploads::Local do
+  let(:data_store) { described_class.new }
+
+  before do
+    stub_uploads_object_storage(FileUploader)
+  end
+
+  context 'model with uploads' do
+    let(:project) { create(:project) }
+    let(:relation) { project.uploads }
+
+    describe '#keys' do
+      let!(:uploads) { create_list(:upload, 2, uploader: FileUploader, model: project) }
+      subject { data_store.keys(relation) }
+
+      it 'returns keys' do
+        is_expected.to match_array(relation.map(&:absolute_path))
+      end
+    end
+
+    describe '#delete_keys' do
+      let(:keys) { data_store.keys(relation) }
+      let!(:uploads) { create_list(:upload, 2, :with_file, :issuable_upload, model: project) }
+      subject { data_store.delete_keys(keys) }
+
+      it 'deletes multiple data' do
+        paths = relation.map(&:absolute_path)
+
+        paths.each do |path|
+          expect(File.exist?(path)).to be_truthy
+        end
+
+        subject
+
+        paths.each do |path|
+          expect(File.exist?(path)).to be_falsey
+        end
+      end
+    end
+  end
+end
diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb
index 6cb27246f06..ff075e65c76 100644
--- a/spec/models/user_spec.rb
+++ b/spec/models/user_spec.rb
@@ -3231,7 +3231,7 @@ describe User do
   end
 
   context 'with uploads' do
-    it_behaves_like 'model with mounted uploader', false do
+    it_behaves_like 'model with uploads', false do
       let(:model_object) { create(:user, :with_avatar) }
       let(:upload_attribute) { :avatar }
       let(:uploader_class) { AttachmentUploader }
diff --git a/spec/support/shared_examples/models/with_uploads_shared_examples.rb b/spec/support/shared_examples/models/with_uploads_shared_examples.rb
index 47ad0c6345d..43033a2d256 100644
--- a/spec/support/shared_examples/models/with_uploads_shared_examples.rb
+++ b/spec/support/shared_examples/models/with_uploads_shared_examples.rb
@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-shared_examples_for 'model with mounted uploader' do |supports_fileuploads|
+shared_examples_for 'model with uploads' do |supports_fileuploads|
   describe '.destroy' do
     before do
       stub_uploads_object_storage(uploader_class)
@@ -8,16 +8,42 @@ shared_examples_for 'model with mounted uploader' do |supports_fileuploads|
       model_object.public_send(upload_attribute).migrate!(ObjectStorage::Store::REMOTE)
     end
 
-    it 'deletes remote uploads' do
-      expect_any_instance_of(CarrierWave::Storage::Fog::File).to receive(:delete).and_call_original
+    context 'with mounted uploader' do
+      it 'deletes remote uploads' do
+        expect_any_instance_of(CarrierWave::Storage::Fog::File).to receive(:delete).and_call_original
 
-      expect { model_object.destroy }.to change { Upload.count }.by(-1)
+        expect { model_object.destroy }.to change { Upload.count }.by(-1)
+      end
     end
 
-    it 'deletes any FileUploader uploads which are not mounted', skip: !supports_fileuploads do
-      create(:upload, uploader: FileUploader, model: model_object)
+    context 'with not mounted uploads', :sidekiq, skip: !supports_fileuploads do
+      context 'with local files' do
+        let!(:uploads) { create_list(:upload, 2, uploader: FileUploader, model: model_object) }
 
-      expect { model_object.destroy }.to change { Upload.count }.by(-2)
+        it 'deletes any FileUploader uploads which are not mounted' do
+          expect { model_object.destroy }.to change { Upload.count }.by(-3)
+        end
+
+        it 'deletes local files' do
+          expect_any_instance_of(Uploads::Local).to receive(:delete_keys).with(uploads.map(&:absolute_path))
+
+          model_object.destroy
+        end
+      end
+
+      context 'with remote files' do
+        let!(:uploads) { create_list(:upload, 2, :object_storage, uploader: FileUploader, model: model_object) }
+
+        it 'deletes any FileUploader uploads which are not mounted' do
+          expect { model_object.destroy }.to change { Upload.count }.by(-3)
+        end
+
+        it 'deletes remote files' do
+          expect_any_instance_of(Uploads::Fog).to receive(:delete_keys).with(uploads.map(&:path))
+
+          model_object.destroy
+        end
+      end
     end
   end
 end