Update Import/Export to use object storage (based on aa feature flag)

5 files changed, 72 insertions, 15 deletions

diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb
index 5ef4e9d530c..15c57a2fc02 100644
--- a/lib/api/project_export.rb
+++ b/lib/api/project_export.rb
@@ -23,9 +23,13 @@ module API
       get ':id/export/download' do
         path = user_project.export_project_path
 
-        render_api_error!('404 Not found or has expired', 404) unless path
-
-        present_disk_file!(path, File.basename(path), 'application/gzip')
+        if path
+          present_disk_file!(path, File.basename(path), 'application/gzip')
+        elsif user_project.export_project_object_exists?
+          present_carrierwave_file!(user_project.import_export_upload.export_file)
+        else
+          render_api_error!('404 Not found or has expired', 404)
+        end
       end
 
       desc 'Start export' do
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index 53fe2f8e436..be3710c5b7f 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -40,6 +40,10 @@ module Gitlab
       "#{basename[0..FILENAME_LIMIT]}_export.tar.gz"
     end
 
+    def object_storage?
+      Feature.enabled?(:import_export_object_storage)
+    end
+
     def version
       VERSION
     end
diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
index aef371d81eb..f0e791e147d 100644
--- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
@@ -24,9 +24,10 @@ module Gitlab
         end
 
         def execute(current_user, project)
-          return unless project&.export_project_path
-
           @project = project
+
+          return unless @project.export_status == :finished
+
           @current_user = current_user
 
           if invalid?
@@ -51,9 +52,12 @@ module Gitlab
         end
 
         def self.lock_file_path(project)
-          return unless project&.export_path
+          return unless project.export_path || object_storage?
 
-          File.join(project.export_path, AFTER_EXPORT_LOCK_FILE_NAME)
+          lock_path = project.import_export_shared.archive_path
+
+          FileUtils.mkdir_p(lock_path)
+          File.join(lock_path, AFTER_EXPORT_LOCK_FILE_NAME)
         end
 
         protected
@@ -77,6 +81,10 @@ module Gitlab
         def log_validation_errors
           errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
         end
+
+        def object_storage?
+          project.export_project_object_exists?
+        end
       end
     end
   end
diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
index 938664a95a1..dce8f89c0ab 100644
--- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
@@ -38,14 +38,20 @@ module Gitlab
         private
 
         def send_file
-          export_file = File.open(project.export_project_path)
-
-          Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options(export_file)) # rubocop:disable GitlabSecurity/PublicSend
+          Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend
         ensure
-          export_file.close if export_file
+          export_file.close if export_file && !object_storage?
+        end
+
+        def export_file
+          if object_storage?
+            project.import_export_upload.export_file.file.open
+          else
+            File.open(project.export_project_path)
+          end
         end
 
-        def send_file_options(export_file)
+        def send_file_options
           {
             body_stream: export_file,
             headers: headers
@@ -53,7 +59,15 @@ module Gitlab
         end
 
         def headers
-          { 'Content-Length' => File.size(project.export_project_path).to_s }
+          { 'Content-Length' => export_size.to_s }
+        end
+
+        def export_size
+          if object_storage?
+            project.import_export_upload.export_file.file.size
+          else
+            File.size(project.export_project_path)
+          end
         end
       end
     end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
index 2daeba90a51..3cd153a4fd2 100644
--- a/lib/gitlab/import_export/saver.rb
+++ b/lib/gitlab/import_export/saver.rb
@@ -15,15 +15,22 @@ module Gitlab
       def save
         if compress_and_save
           remove_export_path
+
           Rails.logger.info("Saved project export #{archive_file}")
-          archive_file
+
+          save_on_object_storage if use_object_storage?
         else
-          @shared.error(Gitlab::ImportExport::Error.new("Unable to save #{archive_file} into #{@shared.export_path}"))
+          @shared.error(Gitlab::ImportExport::Error.new(error_message))
           false
         end
       rescue => e
         @shared.error(e)
         false
+      ensure
+        if use_object_storage?
+          remove_archive
+          remove_export_path
+        end
       end
 
       private
@@ -36,9 +43,29 @@ module Gitlab
         FileUtils.rm_rf(@shared.export_path)
       end
 
+      def remove_archive
+        FileUtils.rm_rf(@shared.archive_path)
+      end
+
       def archive_file
         @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
       end
+
+      def save_on_object_storage
+        upload = ImportExportUpload.find_or_initialize_by(project: @project)
+
+        File.open(archive_file) { |file| upload.export_file = file }
+
+        upload.save!
+      end
+
+      def use_object_storage?
+        Gitlab::ImportExport.object_storage?
+      end
+
+      def error_message
+        "Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}"
+      end
     end
   end
 end