Merge branch '40781-os-to-ce' into 'master'

Bring Object Storage to CE

Closes #4171, #4163, #3370, #2841, and #29203

See merge request gitlab-org/gitlab-ce!17358

6 files changed, 70 insertions, 32 deletions

diff --git a/app/controllers/concerns/send_file_upload.rb b/app/controllers/concerns/send_file_upload.rb
new file mode 100644
index 00000000000..55011c89886
--- /dev/null
+++ b/app/controllers/concerns/send_file_upload.rb
@@ -0,0 +1,17 @@
+module SendFileUpload
+  def send_upload(file_upload, send_params: {}, redirect_params: {}, attachment: nil, disposition: 'attachment')
+    if attachment
+      redirect_params[:query] = { "response-content-disposition" => "#{disposition};filename=#{attachment.inspect}" }
+      send_params.merge!(filename: attachment, disposition: disposition)
+    end
+
+    if file_upload.file_storage?
+      send_file file_upload.path, send_params
+    elsif file_upload.class.proxy_download_enabled?
+      headers.store(*Gitlab::Workhorse.send_url(file_upload.url(**redirect_params)))
+      head :ok
+    else
+      redirect_to file_upload.url(**redirect_params)
+    end
+  end
+end
diff --git a/app/controllers/concerns/uploads_actions.rb b/app/controllers/concerns/uploads_actions.rb
index 3dbfabcae8a..b9b9b6e4e88 100644
--- a/app/controllers/concerns/uploads_actions.rb
+++ b/app/controllers/concerns/uploads_actions.rb
@@ -1,5 +1,6 @@
 module UploadsActions
   include Gitlab::Utils::StrongMemoize
+  include SendFileUpload
 
   UPLOAD_MOUNTS = %w(avatar attachment file logo header_logo).freeze
 
@@ -26,14 +27,11 @@ module UploadsActions
   def show
     return render_404 unless uploader&.exists?
 
-    if uploader.file_storage?
-      disposition = uploader.image_or_video? ? 'inline' : 'attachment'
-      expires_in 0.seconds, must_revalidate: true, private: true
+    expires_in 0.seconds, must_revalidate: true, private: true
 
-      send_file uploader.file.path, disposition: disposition
-    else
-      redirect_to uploader.url
-    end
+    disposition = uploader.image_or_video? ? 'inline' : 'attachment'
+
+    send_upload(uploader, attachment: uploader.filename, disposition: disposition)
   end
 
   private
@@ -62,19 +60,27 @@ module UploadsActions
   end
 
   def build_uploader_from_upload
-    return nil unless params[:secret] && params[:filename]
+    return unless uploader = build_uploader
 
-    upload_path = uploader_class.upload_path(params[:secret], params[:filename])
-    upload = Upload.find_by(uploader: uploader_class.to_s, path: upload_path)
+    upload_paths = uploader.upload_paths(params[:filename])
+    upload = Upload.find_by(uploader: uploader_class.to_s, path: upload_paths)
     upload&.build_uploader
   end
 
   def build_uploader_from_params
+    return unless uploader = build_uploader
+
+    uploader.retrieve_from_store!(params[:filename])
+    uploader
+  end
+
+  def build_uploader
+    return unless params[:secret] && params[:filename]
+
     uploader = uploader_class.new(model, secret: params[:secret])
 
-    return nil unless uploader.model_valid?
+    return unless uploader.model_valid?
 
-    uploader.retrieve_from_store!(params[:filename])
     uploader
   end
 
diff --git a/app/controllers/projects/artifacts_controller.rb b/app/controllers/projects/artifacts_controller.rb
index 0837451cc49..abc283d7aa9 100644
--- a/app/controllers/projects/artifacts_controller.rb
+++ b/app/controllers/projects/artifacts_controller.rb
@@ -1,6 +1,7 @@
 class Projects::ArtifactsController < Projects::ApplicationController
   include ExtractsPath
   include RendersBlob
+  include SendFileUpload
 
   layout 'project'
   before_action :authorize_read_build!
@@ -10,11 +11,7 @@ class Projects::ArtifactsController < Projects::ApplicationController
   before_action :entry, only: [:file]
 
   def download
-    if artifacts_file.file_storage?
-      send_file artifacts_file.path, disposition: 'attachment'
-    else
-      redirect_to artifacts_file.url
-    end
+    send_upload(artifacts_file, attachment: artifacts_file.filename)
   end
 
   def browse
@@ -45,8 +42,7 @@ class Projects::ArtifactsController < Projects::ApplicationController
   end
 
   def raw
-    path = Gitlab::Ci::Build::Artifacts::Path
-      .new(params[:path])
+    path = Gitlab::Ci::Build::Artifacts::Path.new(params[:path])
 
     send_artifacts_entry(build, path)
   end
@@ -75,7 +71,7 @@ class Projects::ArtifactsController < Projects::ApplicationController
   end
 
   def validate_artifacts!
-    render_404 unless build && build.artifacts?
+    render_404 unless build&.artifacts?
   end
 
   def build
diff --git a/app/controllers/projects/jobs_controller.rb b/app/controllers/projects/jobs_controller.rb
index 8b54ba3ad7c..85e972d9731 100644
--- a/app/controllers/projects/jobs_controller.rb
+++ b/app/controllers/projects/jobs_controller.rb
@@ -1,4 +1,6 @@
 class Projects::JobsController < Projects::ApplicationController
+  include SendFileUpload
+
   before_action :build, except: [:index, :cancel_all]
 
   before_action :authorize_read_build!,
@@ -117,11 +119,17 @@ class Projects::JobsController < Projects::ApplicationController
   end
 
   def raw
-    build.trace.read do |stream|
-      if stream.file?
-        send_file stream.path, type: 'text/plain; charset=utf-8', disposition: 'inline'
-      else
-        render_404
+    if trace_artifact_file
+      send_upload(trace_artifact_file,
+                  send_params: raw_send_params,
+                  redirect_params: raw_redirect_params)
+    else
+      build.trace.read do |stream|
+        if stream.file?
+          send_file stream.path, type: 'text/plain; charset=utf-8', disposition: 'inline'
+        else
+          render_404
+        end
       end
     end
   end
@@ -136,9 +144,21 @@ class Projects::JobsController < Projects::ApplicationController
     return access_denied! unless can?(current_user, :erase_build, build)
   end
 
+  def raw_send_params
+    { type: 'text/plain; charset=utf-8', disposition: 'inline' }
+  end
+
+  def raw_redirect_params
+    { query: { 'response-content-type' => 'text/plain; charset=utf-8', 'response-content-disposition' => 'inline' } }
+  end
+
+  def trace_artifact_file
+    @trace_artifact_file ||= build.job_artifacts_trace&.file
+  end
+
   def build
     @build ||= project.builds.find(params[:id])
-      .present(current_user: current_user)
+                 .present(current_user: current_user)
   end
 
   def build_path(build)
diff --git a/app/controllers/projects/lfs_storage_controller.rb b/app/controllers/projects/lfs_storage_controller.rb
index 941638db427..6b16f1ccbbb 100644
--- a/app/controllers/projects/lfs_storage_controller.rb
+++ b/app/controllers/projects/lfs_storage_controller.rb
@@ -1,6 +1,7 @@
 class Projects::LfsStorageController < Projects::GitHttpClientController
   include LfsRequest
   include WorkhorseRequest
+  include SendFileUpload
 
   skip_before_action :verify_workhorse_api!, only: [:download, :upload_finalize]
 
@@ -11,7 +12,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController
       return
     end
 
-    send_file lfs_object.file.path, content_type: "application/octet-stream"
+    send_upload(lfs_object.file, send_params: { content_type: "application/octet-stream" })
   end
 
   def upload_authorize
@@ -70,10 +71,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController
   end
 
   def move_tmp_file_to_storage(object, path)
-    File.open(path) do |f|
-      object.file = f
-    end
-
+    object.file = File.open(path)
     object.file.store!
     object.save
   end
diff --git a/app/controllers/projects/raw_controller.rb b/app/controllers/projects/raw_controller.rb
index a02cc477e08..9bc774b7636 100644
--- a/app/controllers/projects/raw_controller.rb
+++ b/app/controllers/projects/raw_controller.rb
@@ -2,6 +2,7 @@
 class Projects::RawController < Projects::ApplicationController
   include ExtractsPath
   include BlobHelper
+  include SendFileUpload
 
   before_action :require_non_empty_project
   before_action :assign_ref_vars
@@ -31,7 +32,7 @@ class Projects::RawController < Projects::ApplicationController
     lfs_object = find_lfs_object
 
     if lfs_object && lfs_object.project_allowed_access?(@project)
-      send_file lfs_object.file.path, filename: @blob.name, disposition: 'attachment'
+      send_upload(lfs_object.file, attachment: @blob.name)
     else
       render_404
     end