14 files changed, 345 insertions, 178 deletions

diff --git a/app/uploaders/artifact_uploader.rb b/app/uploaders/artifact_uploader.rb
deleted file mode 100644
index 14addb6cf14..00000000000
--- a/app/uploaders/artifact_uploader.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-class ArtifactUploader < GitlabUploader
-  storage :file
-
-  attr_reader :job, :field
-
-  def self.local_artifacts_store
-    Gitlab.config.artifacts.path
-  end
-
-  def self.artifacts_upload_path
-    File.join(self.local_artifacts_store, 'tmp/uploads/')
-  end
-
-  def initialize(job, field)
-    @job, @field = job, field
-  end
-
-  def store_dir
-    default_local_path
-  end
-
-  def cache_dir
-    File.join(self.class.local_artifacts_store, 'tmp/cache')
-  end
-
-  def work_dir
-    File.join(self.class.local_artifacts_store, 'tmp/work')
-  end
-
-  private
-
-  def default_local_path
-    File.join(self.class.local_artifacts_store, default_path)
-  end
-
-  def default_path
-    File.join(job.created_at.utc.strftime('%Y_%m'), job.project_id.to_s, job.id.to_s)
-  end
-end
diff --git a/app/uploaders/attachment_uploader.rb b/app/uploaders/attachment_uploader.rb
index 109eb2fea0b..4930fb2fca7 100644
--- a/app/uploaders/attachment_uploader.rb
+++ b/app/uploaders/attachment_uploader.rb
@@ -1,10 +1,12 @@
 class AttachmentUploader < GitlabUploader
-  include RecordsUploads
   include UploaderHelper
+  include RecordsUploads::Concern
 
   storage :file
 
-  def store_dir
-    "#{base_dir}/#{model.class.to_s.underscore}/#{mounted_as}/#{model.id}"
+  private
+
+  def dynamic_segment
+    File.join(model.class.to_s.underscore, mounted_as.to_s, model.id.to_s)
   end
 end
diff --git a/app/uploaders/avatar_uploader.rb b/app/uploaders/avatar_uploader.rb
index cbb79376d5f..5c8e1cea62e 100644
--- a/app/uploaders/avatar_uploader.rb
+++ b/app/uploaders/avatar_uploader.rb
@@ -1,25 +1,24 @@
 class AvatarUploader < GitlabUploader
-  include RecordsUploads
   include UploaderHelper
+  include RecordsUploads::Concern
 
   storage :file
 
-  def store_dir
-    "#{base_dir}/#{model.class.to_s.underscore}/#{mounted_as}/#{model.id}"
-  end
-
   def exists?
     model.avatar.file && model.avatar.file.present?
   end
 
-  # We set move_to_store and move_to_cache to 'false' to prevent stealing
-  # the avatar file from a project when forking it.
-  # https://gitlab.com/gitlab-org/gitlab-ce/issues/26158
-  def move_to_store
+  def move_to_cache
     false
   end
 
-  def move_to_cache
+  def move_to_store
     false
   end
+
+  private
+
+  def dynamic_segment
+    File.join(model.class.to_s.underscore, mounted_as.to_s, model.id.to_s)
+  end
 end
diff --git a/app/uploaders/file_mover.rb b/app/uploaders/file_mover.rb
index 00c2888d224..8f56f09c9f7 100644
--- a/app/uploaders/file_mover.rb
+++ b/app/uploaders/file_mover.rb
@@ -21,7 +21,8 @@ class FileMover
   end
 
   def update_markdown
-    updated_text = model.read_attribute(update_field).gsub(temp_file_uploader.to_markdown, uploader.to_markdown)
+    updated_text = model.read_attribute(update_field)
+                        .gsub(temp_file_uploader.markdown_link, uploader.markdown_link)
     model.update_attribute(update_field, updated_text)
 
     true
@@ -48,11 +49,11 @@ class FileMover
   end
 
   def uploader
-    @uploader ||= PersonalFileUploader.new(model, secret)
+    @uploader ||= PersonalFileUploader.new(model, secret: secret)
   end
 
   def temp_file_uploader
-    @temp_file_uploader ||= PersonalFileUploader.new(nil, secret)
+    @temp_file_uploader ||= PersonalFileUploader.new(nil, secret: secret)
   end
 
   def revert
diff --git a/app/uploaders/file_uploader.rb b/app/uploaders/file_uploader.rb
index 7027ac4b5db..bde1161dfa8 100644
--- a/app/uploaders/file_uploader.rb
+++ b/app/uploaders/file_uploader.rb
@@ -1,23 +1,40 @@
+# This class breaks the actual CarrierWave concept.
+# Every uploader should use a base_dir that is model agnostic so we can build
+# back URLs from base_dir-relative paths saved in the `Upload` model.
+#
+# As the `.base_dir` is model dependent and **not** saved in the upload model (see #upload_path)
+# there is no way to build back the correct file path without the model, which defies
+# CarrierWave way of storing files.
+#
 class FileUploader < GitlabUploader
-  include RecordsUploads
   include UploaderHelper
+  include RecordsUploads::Concern
 
   MARKDOWN_PATTERN = %r{\!?\[.*?\]\(/uploads/(?<secret>[0-9a-f]{32})/(?<file>.*?)\)}
+  DYNAMIC_PATH_PATTERN = %r{(?<secret>\h{32})/(?<identifier>.*)}
 
   storage :file
 
-  def self.absolute_path(upload_record)
+  after :remove, :prune_store_dir
+
+  def self.root
+    File.join(options.storage_path, 'uploads')
+  end
+
+  def self.absolute_path(upload)
     File.join(
-      self.dynamic_path_segment(upload_record.model),
-      upload_record.path
+      absolute_base_dir(upload.model),
+      upload.path # already contain the dynamic_segment, see #upload_path
     )
   end
 
-  # Not using `GitlabUploader.base_dir` because all project namespaces are in
-  # the `public/uploads` dir.
-  #
-  def self.base_dir
-    root_dir
+  def self.base_dir(model)
+    model_path_segment(model)
+  end
+
+  # used in migrations and import/exports
+  def self.absolute_base_dir(model)
+    File.join(root, base_dir(model))
   end
 
   # Returns the part of `store_dir` that can change based on the model's current
@@ -26,55 +43,119 @@ class FileUploader < GitlabUploader
   # This is used to build Upload paths dynamically based on the model's current
   # namespace and path, allowing us to ignore renames or transfers.
   #
-  # model - Object that responds to `path_with_namespace`
+  # model - Object that responds to `full_path` and `disk_path`
   #
   # Returns a String without a trailing slash
-  def self.dynamic_path_segment(model)
-    File.join(CarrierWave.root, base_dir, model.full_path)
+  def self.model_path_segment(model)
+    if model.hashed_storage?(:attachments)
+      model.disk_path
+    else
+      model.full_path
+    end
+  end
+
+  def self.upload_path(secret, identifier)
+    File.join(secret, identifier)
+  end
+
+  def self.generate_secret
+    SecureRandom.hex
   end
 
   attr_accessor :model
-  attr_reader :secret
 
-  def initialize(model, secret = nil)
+  def initialize(model, mounted_as = nil, **uploader_context)
+    super(model, nil, **uploader_context)
+
     @model = model
-    @secret = secret || generate_secret
+    apply_context!(uploader_context)
   end
 
-  def store_dir
-    File.join(dynamic_path_segment, @secret)
+  def base_dir
+    self.class.base_dir(@model)
   end
 
-  def relative_path
-    self.file.path.sub("#{dynamic_path_segment}/", '')
+  # we don't need to know the actual path, an uploader instance should be
+  # able to yield the file content on demand, so we should build the digest
+  def absolute_path
+    self.class.absolute_path(@upload)
   end
 
-  def to_markdown
-    to_h[:markdown]
+  def upload_path
+    self.class.upload_path(dynamic_segment, identifier)
   end
 
-  def to_h
-    filename = image_or_video? ? self.file.basename : self.file.filename
-    escaped_filename = filename.gsub("]", "\\]")
+  def model_path_segment
+    self.class.model_path_segment(@model)
+  end
 
-    markdown = "[#{escaped_filename}](#{secure_url})"
+  def store_dir
+    File.join(base_dir, dynamic_segment)
+  end
+
+  def markdown_link
+    markdown = "[#{markdown_name}](#{secure_url})"
     markdown.prepend("!") if image_or_video? || dangerous?
+    markdown
+  end
 
+  def to_h
     {
-      alt:      filename,
+      alt:      markdown_name,
       url:      secure_url,
-      markdown: markdown
+      markdown: markdown_link
     }
   end
 
+  def filename
+    self.file.filename
+  end
+
+  def upload=(value)
+    super
+
+    return unless value
+    return if apply_context!(value.uploader_context)
+
+    # fallback to the regex based extraction
+    if matches = DYNAMIC_PATH_PATTERN.match(value.path)
+      @secret = matches[:secret]
+      @identifier = matches[:identifier]
+    end
+  end
+
+  def secret
+    @secret ||= self.class.generate_secret
+  end
+
   private
 
-  def dynamic_path_segment
-    self.class.dynamic_path_segment(model)
+  def apply_context!(uploader_context)
+    @secret, @identifier = uploader_context.values_at(:secret, :identifier)
+
+    !!(@secret && @identifier)
   end
 
-  def generate_secret
-    SecureRandom.hex
+  def build_upload
+    super.tap do |upload|
+      upload.secret = secret
+    end
+  end
+
+  def prune_store_dir
+    storage.delete_dir!(store_dir) # only remove when empty
+  end
+
+  def markdown_name
+    (image_or_video? ? File.basename(filename, File.extname(filename)) : filename).gsub("]", "\\]")
+  end
+
+  def identifier
+    @identifier ||= filename
+  end
+
+  def dynamic_segment
+    secret
   end
 
   def secure_url
diff --git a/app/uploaders/gitlab_uploader.rb b/app/uploaders/gitlab_uploader.rb
index 7f72b3ce471..010100f2da1 100644
--- a/app/uploaders/gitlab_uploader.rb
+++ b/app/uploaders/gitlab_uploader.rb
@@ -1,72 +1,85 @@
 class GitlabUploader < CarrierWave::Uploader::Base
-  def self.absolute_path(upload_record)
-    File.join(CarrierWave.root, upload_record.path)
-  end
+  class_attribute :options
 
-  def self.root_dir
-    'uploads'
-  end
+  class << self
+    # DSL setter
+    def storage_options(options)
+      self.options = options
+    end
 
-  # When object storage is used, keep the `root_dir` as `base_dir`.
-  # The files aren't really in folders there, they just have a name.
-  # The files that contain user input in their name, also contain a hash, so
-  # the names are still unique
-  #
-  # This method is overridden in the `FileUploader`
-  def self.base_dir
-    return root_dir unless file_storage?
+    def root
+      options.storage_path
+    end
 
-    File.join(root_dir, '-', 'system')
-  end
+    # represent the directory namespacing at the class level
+    def base_dir
+      options.fetch('base_dir', '')
+    end
 
-  def self.file_storage?
-    self.storage == CarrierWave::Storage::File
+    def file_storage?
+      storage == CarrierWave::Storage::File
+    end
+
+    def absolute_path(upload_record)
+      File.join(root, upload_record.path)
+    end
   end
 
+  storage_options Gitlab.config.uploads
+
   delegate :base_dir, :file_storage?, to: :class
 
+  def initialize(model, mounted_as = nil, **uploader_context)
+    super(model, mounted_as)
+  end
+
   def file_cache_storage?
     cache_storage.is_a?(CarrierWave::Storage::File)
   end
 
   # Reduce disk IO
   def move_to_cache
-    true
+    file_storage?
   end
 
   # Reduce disk IO
   def move_to_store
-    true
-  end
-
-  # Designed to be overridden by child uploaders that have a dynamic path
-  # segment -- that is, a path that changes based on mutable attributes of its
-  # associated model
-  #
-  # For example, `FileUploader` builds the storage path based on the associated
-  # project model's `path_with_namespace` value, which can change when the
-  # project or its containing namespace is moved or renamed.
-  def relative_path
-    self.file.path.sub("#{root}/", '')
+    file_storage?
   end
 
   def exists?
     file.present?
   end
 
-  # Override this if you don't want to save files by default to the Rails.root directory
+  def store_dir
+    File.join(base_dir, dynamic_segment)
+  end
+
+  def cache_dir
+    File.join(root, base_dir, 'tmp/cache')
+  end
+
   def work_dir
-    # Default path set by CarrierWave:
-    # https://github.com/carrierwaveuploader/carrierwave/blob/v1.0.0/lib/carrierwave/uploader/cache.rb#L182
-    CarrierWave.tmp_path
+    File.join(root, base_dir, 'tmp/work')
   end
 
   def filename
     super || file&.filename
   end
 
+  def model_valid?
+    !!model
+  end
+
   private
 
+  # Designed to be overridden by child uploaders that have a dynamic path
+  # segment -- that is, a path that changes based on mutable attributes of its
+  # associated model
+  def dynamic_segment
+    raise(NotImplementedError)
+  end
+
   # To prevent files from moving across filesystems, override the default
   # implementation:
   # http://github.com/carrierwaveuploader/carrierwave/blob/v1.0.0/lib/carrierwave/uploader/cache.rb#L181-L183
@@ -74,6 +87,6 @@ class GitlabUploader < CarrierWave::Uploader::Base
     # To be safe, keep this directory outside of the the cache directory
     # because calling CarrierWave.clean_cache_files! will remove any files in
     # the cache directory.
-    File.join(work_dir, @cache_id, version_name.to_s, for_file)
+    File.join(work_dir, cache_id, version_name.to_s, for_file)
   end
 end
diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb
new file mode 100644
index 00000000000..ad5385f45a4
--- /dev/null
+++ b/app/uploaders/job_artifact_uploader.rb
@@ -0,0 +1,34 @@
+class JobArtifactUploader < GitlabUploader
+  extend Workhorse::UploadPath
+
+  storage_options Gitlab.config.artifacts
+
+  def size
+    return super if model.size.nil?
+
+    model.size
+  end
+
+  def store_dir
+    dynamic_segment
+  end
+
+  def open
+    raise 'Only File System is supported' unless file_storage?
+
+    File.open(path, "rb") if path
+  end
+
+  private
+
+  def dynamic_segment
+    creation_date = model.created_at.utc.strftime('%Y_%m_%d')
+
+    File.join(disk_hash[0..1], disk_hash[2..3], disk_hash,
+              creation_date, model.job_id.to_s, model.id.to_s)
+  end
+
+  def disk_hash
+    @disk_hash ||= Digest::SHA2.hexdigest(model.project_id.to_s)
+  end
+end
diff --git a/app/uploaders/legacy_artifact_uploader.rb b/app/uploaders/legacy_artifact_uploader.rb
new file mode 100644
index 00000000000..28c458d3ff1
--- /dev/null
+++ b/app/uploaders/legacy_artifact_uploader.rb
@@ -0,0 +1,15 @@
+class LegacyArtifactUploader < GitlabUploader
+  extend Workhorse::UploadPath
+
+  storage_options Gitlab.config.artifacts
+
+  def store_dir
+    dynamic_segment
+  end
+
+  private
+
+  def dynamic_segment
+    File.join(model.created_at.utc.strftime('%Y_%m'), model.project_id.to_s, model.id.to_s)
+  end
+end
diff --git a/app/uploaders/lfs_object_uploader.rb b/app/uploaders/lfs_object_uploader.rb
index d11ebf0f9ca..e04c97ce179 100644
--- a/app/uploaders/lfs_object_uploader.rb
+++ b/app/uploaders/lfs_object_uploader.rb
@@ -1,19 +1,24 @@
 class LfsObjectUploader < GitlabUploader
-  storage :file
+  extend Workhorse::UploadPath
 
-  def store_dir
-    "#{Gitlab.config.lfs.storage_path}/#{model.oid[0, 2]}/#{model.oid[2, 2]}"
+  # LfsObject are in `tmp/upload` instead of `tmp/uploads`
+  def self.workhorse_upload_path
+    File.join(root, 'tmp/upload')
   end
 
-  def cache_dir
-    "#{Gitlab.config.lfs.storage_path}/tmp/cache"
-  end
+  storage_options Gitlab.config.lfs
 
   def filename
     model.oid[4..-1]
   end
 
-  def work_dir
-    File.join(Gitlab.config.lfs.storage_path, 'tmp', 'work')
+  def store_dir
+    dynamic_segment
+  end
+
+  private
+
+  def dynamic_segment
+    File.join(model.oid[0, 2], model.oid[2, 2])
   end
 end
diff --git a/app/uploaders/namespace_file_uploader.rb b/app/uploaders/namespace_file_uploader.rb
new file mode 100644
index 00000000000..993e85fbc13
--- /dev/null
+++ b/app/uploaders/namespace_file_uploader.rb
@@ -0,0 +1,19 @@
+class NamespaceFileUploader < FileUploader
+  # Re-Override
+  def self.root
+    options.storage_path
+  end
+
+  def self.base_dir(model)
+    File.join(options.base_dir, 'namespace', model_path_segment(model))
+  end
+
+  def self.model_path_segment(model)
+    File.join(model.id.to_s)
+  end
+
+  # Re-Override
+  def store_dir
+    File.join(base_dir, dynamic_segment)
+  end
+end
diff --git a/app/uploaders/personal_file_uploader.rb b/app/uploaders/personal_file_uploader.rb
index 3298ad104ec..f2ad0badd53 100644
--- a/app/uploaders/personal_file_uploader.rb
+++ b/app/uploaders/personal_file_uploader.rb
@@ -1,23 +1,33 @@
 class PersonalFileUploader < FileUploader
-  def self.dynamic_path_segment(model)
-    File.join(CarrierWave.root, model_path(model))
+  # Re-Override
+  def self.root
+    options.storage_path
   end
 
-  def self.base_dir
-    File.join(root_dir, '-', 'system')
+  def self.base_dir(model)
+    File.join(options.base_dir, model_path_segment(model))
   end
 
-  private
+  def self.model_path_segment(model)
+    return 'temp/' unless model
 
-  def secure_url
-    File.join(self.class.model_path(model), secret, file.filename)
+    File.join(model.class.to_s.underscore, model.id.to_s)
   end
 
-  def self.model_path(model)
-    if model
-      File.join("/#{base_dir}", model.class.to_s.underscore, model.id.to_s)
-    else
-      File.join("/#{base_dir}", 'temp')
-    end
+  # model_path_segment does not require a model to be passed, so we can always
+  # generate a path, even when there's no model.
+  def model_valid?
+    true
+  end
+
+  # Revert-Override
+  def store_dir
+    File.join(base_dir, dynamic_segment)
+  end
+
+  private
+
+  def secure_url
+    File.join('/', base_dir, secret, file.filename)
   end
 end
diff --git a/app/uploaders/records_uploads.rb b/app/uploaders/records_uploads.rb
index feb4f04d7b7..458928bc067 100644
--- a/app/uploaders/records_uploads.rb
+++ b/app/uploaders/records_uploads.rb
@@ -1,35 +1,62 @@
 module RecordsUploads
-  extend ActiveSupport::Concern
+  module Concern
+    extend ActiveSupport::Concern
 
-  included do
-    after :store,   :record_upload
-    before :remove, :destroy_upload
-  end
+    attr_accessor :upload
 
-  # After storing an attachment, create a corresponding Upload record
-  #
-  # NOTE: We're ignoring the argument passed to this callback because we want
-  # the `SanitizedFile` object from `CarrierWave::Uploader::Base#file`, not the
-  # `Tempfile` object the callback gets.
-  #
-  # Called `after :store`
-  def record_upload(_tempfile = nil)
-    return unless model
-    return unless file_storage?
-    return unless file.exists?
-
-    Upload.record(self)
-  end
+    included do
+      after  :store,  :record_upload
+      before :remove, :destroy_upload
+    end
+
+    # After storing an attachment, create a corresponding Upload record
+    #
+    # NOTE: We're ignoring the argument passed to this callback because we want
+    # the `SanitizedFile` object from `CarrierWave::Uploader::Base#file`, not the
+    # `Tempfile` object the callback gets.
+    #
+    # Called `after :store`
+    def record_upload(_tempfile = nil)
+      return unless model
+      return unless file && file.exists?
+
+      Upload.transaction do
+        uploads.where(path: upload_path).delete_all
+        upload.destroy! if upload
+
+        self.upload = build_upload
+        upload.save!
+      end
+    end
+
+    def upload_path
+      File.join(store_dir, filename.to_s)
+    end
+
+    private
+
+    def uploads
+      Upload.order(id: :desc).where(uploader: self.class.to_s)
+    end
 
-  private
+    def build_upload
+      Upload.new(
+        uploader: self.class.to_s,
+        size: file.size,
+        path: upload_path,
+        model: model,
+        mount_point: mounted_as
+      )
+    end
 
-  # Before removing an attachment, destroy any Upload records at the same path
-  #
-  # Called `before :remove`
-  def destroy_upload(*args)
-    return unless file_storage?
-    return unless file
+    # Before removing an attachment, destroy any Upload records at the same path
+    #
+    # Called `before :remove`
+    def destroy_upload(*args)
+      return unless file && file.exists?
 
-    Upload.remove_path(relative_path)
+      self.upload = nil
+      uploads.where(path: upload_path).delete_all
+    end
   end
 end
diff --git a/app/uploaders/uploader_helper.rb b/app/uploaders/uploader_helper.rb
index 7635c20ab3a..fd446d31092 100644
--- a/app/uploaders/uploader_helper.rb
+++ b/app/uploaders/uploader_helper.rb
@@ -32,14 +32,7 @@ module UploaderHelper
   def extension_match?(extensions)
     return false unless file
 
-    extension =
-      if file.respond_to?(:extension)
-        file.extension
-      else
-        # Not all CarrierWave storages respond to :extension
-        File.extname(file.path).delete('.')
-      end
-
+    extension = file.try(:extension) || File.extname(file.path).delete('.')
     extensions.include?(extension.downcase)
   end
 end
diff --git a/app/uploaders/workhorse.rb b/app/uploaders/workhorse.rb
new file mode 100644
index 00000000000..782032cf516
--- /dev/null
+++ b/app/uploaders/workhorse.rb
@@ -0,0 +1,7 @@
+module Workhorse
+  module UploadPath
+    def workhorse_upload_path
+      File.join(root, base_dir, 'tmp/uploads')
+    end
+  end
+end