From 239fdc78b1ced1861cdcf00b8927963e30ef2095 Mon Sep 17 00:00:00 2001
From: Jan Provaznik <jprovaznik@gitlab.com>
Date: Sun, 7 Oct 2018 20:31:08 +0200
Subject: Use FastDestroy for deleting uploads

It gathers list of file paths to delete before destroying
the parent object. Then after the parent_object is destroyed
these paths are scheduled for deletion asynchronously.

Carrierwave needed associated model for deleting upload file.
To avoid this requirement, simple Fog/File layer is used directly
for file deletion, this allows us to use just a simple list of paths.
---
 app/models/uploads/base.rb  | 19 +++++++++++++++
 app/models/uploads/fog.rb   | 43 ++++++++++++++++++++++++++++++++++
 app/models/uploads/local.rb | 56 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 118 insertions(+)
 create mode 100644 app/models/uploads/base.rb
 create mode 100644 app/models/uploads/fog.rb
 create mode 100644 app/models/uploads/local.rb

(limited to 'app/models/uploads')

diff --git a/app/models/uploads/base.rb b/app/models/uploads/base.rb
new file mode 100644
index 00000000000..f9814159958
--- /dev/null
+++ b/app/models/uploads/base.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Uploads
+  class Base
+    BATCH_SIZE = 100
+
+    attr_reader :logger
+
+    def initialize(logger: nil)
+      @logger ||= Rails.logger
+    end
+
+    def delete_keys_async(keys_to_delete)
+      keys_to_delete.each_slice(BATCH_SIZE) do |batch|
+        DeleteStoredFilesWorker.perform_async(self.class, batch)
+      end
+    end
+  end
+end
diff --git a/app/models/uploads/fog.rb b/app/models/uploads/fog.rb
new file mode 100644
index 00000000000..b44e273e9ab
--- /dev/null
+++ b/app/models/uploads/fog.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Uploads
+  class Fog < Base
+    include ::Gitlab::Utils::StrongMemoize
+
+    def available?
+      object_store.enabled
+    end
+
+    def keys(relation)
+      return [] unless available?
+
+      relation.pluck(:path)
+    end
+
+    def delete_keys(keys)
+      keys.each do |key|
+        connection.delete_object(bucket_name, key)
+      end
+    end
+
+    private
+
+    def object_store
+      Gitlab.config.uploads.object_store
+    end
+
+    def bucket_name
+      return unless available?
+
+      object_store.remote_directory
+    end
+
+    def connection
+      return unless available?
+
+      strong_memoize(:connection) do
+        ::Fog::Storage.new(object_store.connection.to_hash.deep_symbolize_keys)
+      end
+    end
+  end
+end
diff --git a/app/models/uploads/local.rb b/app/models/uploads/local.rb
new file mode 100644
index 00000000000..ffbf840a9d2
--- /dev/null
+++ b/app/models/uploads/local.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Uploads
+  class Local < Base
+    def keys(relation)
+      relation.includes(:model).find_each.map {|u| u.absolute_path }
+    end
+
+    def delete_keys(keys)
+      keys.each do |path|
+        delete_file(path)
+      end
+    end
+
+    private
+
+    def delete_file(path)
+      unless exists?(path)
+        logger.warn("File '#{path}' doesn't exist, skipping")
+        return
+      end
+
+      unless in_uploads?(path)
+        message = "Path '#{path}' is not in uploads dir, skipping"
+        logger.warn(message)
+        Gitlab::Sentry.track_exception(RuntimeError.new(message), extra: { uploads_dir: storage_dir })
+        return
+      end
+
+      FileUtils.rm(path)
+      delete_dir!(File.dirname(path))
+    end
+
+    def exists?(path)
+      path.present? && File.exist?(path)
+    end
+
+    def in_uploads?(path)
+      path.start_with?(storage_dir)
+    end
+
+    def delete_dir!(path)
+      Dir.rmdir(path)
+    rescue Errno::ENOENT
+      # Ignore: path does not exist
+    rescue Errno::ENOTDIR
+      # Ignore: path is not a dir
+    rescue Errno::ENOTEMPTY, Errno::EEXIST
+      # Ignore: dir is not empty
+    end
+
+    def storage_dir
+      @storage_dir ||= File.realpath(Gitlab.config.uploads.storage_path)
+    end
+  end
+end
-- 
cgit v1.2.1