Update Import/Export to use object storage (based on aa feature flag)

6 files changed, 64 insertions, 14 deletions

diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb
index ec3a5788ba1..f2abe27f60e 100644
--- a/app/controllers/projects_controller.rb
+++ b/app/controllers/projects_controller.rb
@@ -2,6 +2,7 @@ class ProjectsController < Projects::ApplicationController
   include IssuableCollections
   include ExtractsPath
   include PreviewMarkdown
+  include SendFileUpload
 
   before_action :whitelist_query_limiting, only: [:create]
   before_action :authenticate_user!, except: [:index, :show, :activity, :refs]
@@ -188,9 +189,9 @@ class ProjectsController < Projects::ApplicationController
   end
 
   def download_export
-    export_project_path = @project.export_project_path
-
-    if export_project_path
+    if export_project_object_storage?
+      send_upload(@project.import_export_upload.export_file)
+    elsif export_project_path
       send_file export_project_path, disposition: 'attachment'
     else
       redirect_to(
@@ -265,8 +266,6 @@ class ProjectsController < Projects::ApplicationController
     render json: options.to_json
   end
 
-  private
-
   # Render project landing depending of which features are available
   # So if page is not availble in the list it renders the next page
   #
@@ -424,4 +423,12 @@ class ProjectsController < Projects::ApplicationController
   def whitelist_query_limiting
     Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440')
   end
+
+  def export_project_path
+    @export_project_path ||= @project.export_project_path
+  end
+
+  def export_project_object_storage?
+    @project.export_project_object_exists?
+  end
 end
diff --git a/app/models/import_export_upload.rb b/app/models/import_export_upload.rb
new file mode 100644
index 00000000000..60d53d6c2c8
--- /dev/null
+++ b/app/models/import_export_upload.rb
@@ -0,0 +1,13 @@
+class ImportExportUpload < ActiveRecord::Base
+  include WithUploads
+  include ObjectStorage::BackgroundMove
+
+  belongs_to :project
+
+  mount_uploader :import_file, ImportExportUploader
+  mount_uploader :export_file, ImportExportUploader
+
+  def retrieve_upload(_identifier, paths)
+    Upload.find_by(model: self, path: paths)
+  end
+end
diff --git a/app/models/project.rb b/app/models/project.rb
index 8f40470de82..770262f6193 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -171,6 +171,7 @@ class Project < ActiveRecord::Base
   has_one :fork_network, through: :fork_network_member
 
   has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project
+  has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
   # Merge Requests for target project should be removed with it
   has_many :merge_requests, foreign_key: 'target_project_id'
@@ -1712,7 +1713,7 @@ class Project < ActiveRecord::Base
       :started
     elsif after_export_in_progress?
       :after_export_action
-    elsif export_project_path
+    elsif export_project_path || export_project_object_exists?
       :finished
     else
       :none
@@ -1727,16 +1728,21 @@ class Project < ActiveRecord::Base
     import_export_shared.after_export_in_progress?
   end
 
-  def remove_exports
-    return nil unless export_path.present?
-
-    FileUtils.rm_rf(export_path)
+  def remove_exports(path = export_path)
+    if path.present?
+      FileUtils.rm_rf(path)
+    elsif export_project_object_exists?
+      import_export_upload.remove_export_file!
+      import_export_upload.save
+    end
   end
 
   def remove_exported_project_file
-    return unless export_project_path.present?
+    remove_exports(export_project_path)
+  end
 
-    FileUtils.rm_f(export_project_path)
+  def export_project_object_exists?
+    Gitlab::ImportExport.object_storage? && import_export_upload&.export_file&.file
   end
 
   def full_path_slug
diff --git a/app/services/import_export_clean_up_service.rb b/app/services/import_export_clean_up_service.rb
index 74088b970c9..3702c3742ef 100644
--- a/app/services/import_export_clean_up_service.rb
+++ b/app/services/import_export_clean_up_service.rb
@@ -10,7 +10,9 @@ class ImportExportCleanUpService
 
   def execute
     Gitlab::Metrics.measure(:import_export_clean_up) do
-      next unless File.directory?(path)
+      clean_up_export_object_files
+
+      break unless File.directory?(path)
 
       clean_up_export_files
     end
@@ -21,4 +23,11 @@ class ImportExportCleanUpService
   def clean_up_export_files
     Gitlab::Popen.popen(%W(find #{path} -not -path #{path} -mmin +#{mmin} -delete))
   end
+
+  def clean_up_export_object_files
+    ImportExportUpload.where('updated_at < ?', mmin.minutes.ago).each do |upload|
+      upload.remove_export_file!
+      upload.save!
+    end
+  end
 end
diff --git a/app/uploaders/import_export_uploader.rb b/app/uploaders/import_export_uploader.rb
new file mode 100644
index 00000000000..213ac5c8011
--- /dev/null
+++ b/app/uploaders/import_export_uploader.rb
@@ -0,0 +1,15 @@
+class ImportExportUploader < AttachmentUploader
+  EXTENSION_WHITELIST = %w[tar.gz].freeze
+
+  def extension_whitelist
+    EXTENSION_WHITELIST
+  end
+
+  def move_to_store
+    true
+  end
+
+  def move_to_cache
+    false
+  end
+end
diff --git a/app/views/projects/_export.html.haml b/app/views/projects/_export.html.haml
index f4d4888bd15..aa980da7e95 100644
--- a/app/views/projects/_export.html.haml
+++ b/app/views/projects/_export.html.haml
@@ -31,7 +31,7 @@
           %li Any encrypted tokens
     %p
       Once the exported file is ready, you will receive a notification email with a download link, or you can download it from this page.
-    - if project.export_project_path
+    - if project.export_status == :finished
       = link_to 'Download export',  download_export_project_path(project),
               rel: 'nofollow', download: '', method: :get, class: "btn btn-default"
       = link_to 'Generate new export',  generate_new_export_project_path(project),