summaryrefslogtreecommitdiff
path: root/app
diff options
context:
space:
mode:
authorJames Lopez <james@jameslopez.es>2018-06-25 15:10:26 +0200
committerJames Lopez <james@jameslopez.es>2018-07-06 15:46:18 +0200
commita2bf1641546a1d3eeb3e9f44734854f655c0adef (patch)
tree0652c20a92513330aa09c4a2ec9adbfaeb3a6494 /app
parentb0fa01fce3822da94aee6264829841996beb6df3 (diff)
downloadgitlab-ce-a2bf1641546a1d3eeb3e9f44734854f655c0adef.tar.gz
Update Import/Export to use object storage (based on aa feature flag)
Diffstat (limited to 'app')
-rw-r--r--app/controllers/projects_controller.rb17
-rw-r--r--app/models/import_export_upload.rb13
-rw-r--r--app/models/project.rb20
-rw-r--r--app/services/import_export_clean_up_service.rb11
-rw-r--r--app/uploaders/import_export_uploader.rb15
-rw-r--r--app/views/projects/_export.html.haml2
6 files changed, 64 insertions, 14 deletions
diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb
index ec3a5788ba1..f2abe27f60e 100644
--- a/app/controllers/projects_controller.rb
+++ b/app/controllers/projects_controller.rb
@@ -2,6 +2,7 @@ class ProjectsController < Projects::ApplicationController
include IssuableCollections
include ExtractsPath
include PreviewMarkdown
+ include SendFileUpload
before_action :whitelist_query_limiting, only: [:create]
before_action :authenticate_user!, except: [:index, :show, :activity, :refs]
@@ -188,9 +189,9 @@ class ProjectsController < Projects::ApplicationController
end
def download_export
- export_project_path = @project.export_project_path
-
- if export_project_path
+ if export_project_object_storage?
+ send_upload(@project.import_export_upload.export_file)
+ elsif export_project_path
send_file export_project_path, disposition: 'attachment'
else
redirect_to(
@@ -265,8 +266,6 @@ class ProjectsController < Projects::ApplicationController
render json: options.to_json
end
- private
-
# Render project landing depending of which features are available
# So if page is not availble in the list it renders the next page
#
@@ -424,4 +423,12 @@ class ProjectsController < Projects::ApplicationController
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440')
end
+
+ def export_project_path
+ @export_project_path ||= @project.export_project_path
+ end
+
+ def export_project_object_storage?
+ @project.export_project_object_exists?
+ end
end
diff --git a/app/models/import_export_upload.rb b/app/models/import_export_upload.rb
new file mode 100644
index 00000000000..60d53d6c2c8
--- /dev/null
+++ b/app/models/import_export_upload.rb
@@ -0,0 +1,13 @@
+class ImportExportUpload < ActiveRecord::Base
+ include WithUploads
+ include ObjectStorage::BackgroundMove
+
+ belongs_to :project
+
+ mount_uploader :import_file, ImportExportUploader
+ mount_uploader :export_file, ImportExportUploader
+
+ def retrieve_upload(_identifier, paths)
+ Upload.find_by(model: self, path: paths)
+ end
+end
diff --git a/app/models/project.rb b/app/models/project.rb
index 8f40470de82..770262f6193 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -171,6 +171,7 @@ class Project < ActiveRecord::Base
has_one :fork_network, through: :fork_network_member
has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project
+ has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
# Merge Requests for target project should be removed with it
has_many :merge_requests, foreign_key: 'target_project_id'
@@ -1712,7 +1713,7 @@ class Project < ActiveRecord::Base
:started
elsif after_export_in_progress?
:after_export_action
- elsif export_project_path
+ elsif export_project_path || export_project_object_exists?
:finished
else
:none
@@ -1727,16 +1728,21 @@ class Project < ActiveRecord::Base
import_export_shared.after_export_in_progress?
end
- def remove_exports
- return nil unless export_path.present?
-
- FileUtils.rm_rf(export_path)
+ def remove_exports(path = export_path)
+ if path.present?
+ FileUtils.rm_rf(path)
+ elsif export_project_object_exists?
+ import_export_upload.remove_export_file!
+ import_export_upload.save
+ end
end
def remove_exported_project_file
- return unless export_project_path.present?
+ remove_exports(export_project_path)
+ end
- FileUtils.rm_f(export_project_path)
+ def export_project_object_exists?
+ Gitlab::ImportExport.object_storage? && import_export_upload&.export_file&.file
end
def full_path_slug
diff --git a/app/services/import_export_clean_up_service.rb b/app/services/import_export_clean_up_service.rb
index 74088b970c9..3702c3742ef 100644
--- a/app/services/import_export_clean_up_service.rb
+++ b/app/services/import_export_clean_up_service.rb
@@ -10,7 +10,9 @@ class ImportExportCleanUpService
def execute
Gitlab::Metrics.measure(:import_export_clean_up) do
- next unless File.directory?(path)
+ clean_up_export_object_files
+
+ break unless File.directory?(path)
clean_up_export_files
end
@@ -21,4 +23,11 @@ class ImportExportCleanUpService
def clean_up_export_files
Gitlab::Popen.popen(%W(find #{path} -not -path #{path} -mmin +#{mmin} -delete))
end
+
+ def clean_up_export_object_files
+ ImportExportUpload.where('updated_at < ?', mmin.minutes.ago).each do |upload|
+ upload.remove_export_file!
+ upload.save!
+ end
+ end
end
diff --git a/app/uploaders/import_export_uploader.rb b/app/uploaders/import_export_uploader.rb
new file mode 100644
index 00000000000..213ac5c8011
--- /dev/null
+++ b/app/uploaders/import_export_uploader.rb
@@ -0,0 +1,15 @@
+class ImportExportUploader < AttachmentUploader
+ EXTENSION_WHITELIST = %w[tar.gz].freeze
+
+ def extension_whitelist
+ EXTENSION_WHITELIST
+ end
+
+ def move_to_store
+ true
+ end
+
+ def move_to_cache
+ false
+ end
+end
diff --git a/app/views/projects/_export.html.haml b/app/views/projects/_export.html.haml
index f4d4888bd15..aa980da7e95 100644
--- a/app/views/projects/_export.html.haml
+++ b/app/views/projects/_export.html.haml
@@ -31,7 +31,7 @@
%li Any encrypted tokens
%p
Once the exported file is ready, you will receive a notification email with a download link, or you can download it from this page.
- - if project.export_project_path
+ - if project.export_status == :finished
= link_to 'Download export', download_export_project_path(project),
rel: 'nofollow', download: '', method: :get, class: "btn btn-default"
= link_to 'Generate new export', generate_new_export_project_path(project),