summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDouwe Maan <douwe@gitlab.com>2018-03-30 15:46:00 +0000
committerDouwe Maan <douwe@gitlab.com>2018-03-30 15:46:00 +0000
commit96cd05e65ef6b3b98734f041a43fc8cc81dba9b5 (patch)
treee2e1cff25e9e4ab67252b0402cb5df95fdc98d25
parent7c36e8561c60882e6b0b47c563f7d19f3d6b02a6 (diff)
parent22b05a1ff74d4f64490f93995259602b3d07c3cf (diff)
downloadgitlab-ce-96cd05e65ef6b3b98734f041a43fc8cc81dba9b5.tar.gz
Merge branch 'fj-42685-extend-project-export-endpoint' into 'master'
Extend API for exporting a project with direct upload URL Closes #42685 See merge request gitlab-org/gitlab-ce!17686
-rw-r--r--app/models/project.rb16
-rw-r--r--app/services/projects/import_export/export_service.rb33
-rw-r--r--app/workers/project_export_worker.rb14
-rw-r--r--changelogs/unreleased/fj-42685-extend-project-export-endpoint.yml5
-rw-r--r--doc/api/project_import_export.md19
-rw-r--r--lib/api/project_export.rb19
-rw-r--r--lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb83
-rw-r--r--lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb17
-rw-r--r--lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb61
-rw-r--r--lib/gitlab/import_export/after_export_strategy_builder.rb24
-rw-r--r--lib/gitlab/import_export/shared.rb14
-rw-r--r--spec/fixtures/api/schemas/public_api/v4/project/export_status.json11
-rw-r--r--spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb104
-rw-r--r--spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb36
-rw-r--r--spec/lib/gitlab/import_export/after_export_strategy_builder_spec.rb29
-rw-r--r--spec/models/project_spec.rb19
-rw-r--r--spec/requests/api/project_export_spec.rb79
-rw-r--r--spec/services/projects/import_export/export_service_spec.rb85
-rw-r--r--spec/workers/project_export_worker_spec.rb28
19 files changed, 671 insertions, 25 deletions
diff --git a/app/models/project.rb b/app/models/project.rb
index da03080f440..b343786d2c9 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -1544,8 +1544,8 @@ class Project < ActiveRecord::Base
@errors = original_errors
end
- def add_export_job(current_user:, params: {})
- job_id = ProjectExportWorker.perform_async(current_user.id, self.id, params)
+ def add_export_job(current_user:, after_export_strategy: nil, params: {})
+ job_id = ProjectExportWorker.perform_async(current_user.id, self.id, after_export_strategy, params)
if job_id
Rails.logger.info "Export job started for project ID #{self.id} with job ID #{job_id}"
@@ -1571,6 +1571,8 @@ class Project < ActiveRecord::Base
def export_status
if export_in_progress?
:started
+ elsif after_export_in_progress?
+ :after_export_action
elsif export_project_path
:finished
else
@@ -1582,12 +1584,22 @@ class Project < ActiveRecord::Base
import_export_shared.active_export_count > 0
end
+ def after_export_in_progress?
+ import_export_shared.after_export_in_progress?
+ end
+
def remove_exports
return nil unless export_path.present?
FileUtils.rm_rf(export_path)
end
+ def remove_exported_project_file
+ return unless export_project_path.present?
+
+ FileUtils.rm_f(export_project_path)
+ end
+
def full_path_slug
Gitlab::Utils.slugify(full_path.to_s)
end
diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb
index d16aa3de639..402cddd3ec1 100644
--- a/app/services/projects/import_export/export_service.rb
+++ b/app/services/projects/import_export/export_service.rb
@@ -1,22 +1,36 @@
module Projects
module ImportExport
class ExportService < BaseService
- def execute(_options = {})
+ def execute(after_export_strategy = nil, options = {})
@shared = project.import_export_shared
- save_all
+
+ save_all!
+ execute_after_export_action(after_export_strategy)
end
private
- def save_all
- if [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save)
+ def execute_after_export_action(after_export_strategy)
+ return unless after_export_strategy
+
+ unless after_export_strategy.execute(current_user, project)
+ cleanup_and_notify_error
+ end
+ end
+
+ def save_all!
+ if save_services
Gitlab::ImportExport::Saver.save(project: project, shared: @shared)
notify_success
else
- cleanup_and_notify
+ cleanup_and_notify_error!
end
end
+ def save_services
+ [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save)
+ end
+
def version_saver
Gitlab::ImportExport::VersionSaver.new(shared: @shared)
end
@@ -41,19 +55,22 @@ module Projects
Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared)
end
- def cleanup_and_notify
+ def cleanup_and_notify_error
Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{@shared.errors.join(', ')}")
FileUtils.rm_rf(@shared.export_path)
notify_error
+ end
+
+ def cleanup_and_notify_error!
+ cleanup_and_notify_error
+
raise Gitlab::ImportExport::Error.new(@shared.errors.join(', '))
end
def notify_success
Rails.logger.info("Import/Export - Project #{project.name} with ID: #{project.id} successfully exported")
-
- notification_service.project_exported(@project, @current_user)
end
def notify_error
diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb
index 0b502143e5d..c3d84bb0b93 100644
--- a/app/workers/project_export_worker.rb
+++ b/app/workers/project_export_worker.rb
@@ -4,11 +4,19 @@ class ProjectExportWorker
sidekiq_options retry: 3
- def perform(current_user_id, project_id, params = {})
- params = params.with_indifferent_access
+ def perform(current_user_id, project_id, after_export_strategy = {}, params = {})
current_user = User.find(current_user_id)
project = Project.find(project_id)
+ after_export = build!(after_export_strategy)
- ::Projects::ImportExport::ExportService.new(project, current_user, params).execute
+ ::Projects::ImportExport::ExportService.new(project, current_user, params).execute(after_export)
+ end
+
+ private
+
+ def build!(after_export_strategy)
+ strategy_klass = after_export_strategy&.delete('klass')
+
+ Gitlab::ImportExport::AfterExportStrategyBuilder.build!(strategy_klass, after_export_strategy)
end
end
diff --git a/changelogs/unreleased/fj-42685-extend-project-export-endpoint.yml b/changelogs/unreleased/fj-42685-extend-project-export-endpoint.yml
new file mode 100644
index 00000000000..a06499d821a
--- /dev/null
+++ b/changelogs/unreleased/fj-42685-extend-project-export-endpoint.yml
@@ -0,0 +1,5 @@
+---
+title: Extend API for exporting a project with direct upload URL
+merge_request: 17686
+author:
+type: added
diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md
index de5207fc5e4..5467187788a 100644
--- a/doc/api/project_import_export.md
+++ b/doc/api/project_import_export.md
@@ -8,6 +8,14 @@
Start a new export.
+The endpoint also accepts an `upload` param. This param is a hash that contains
+all the necessary information to upload the exported project to a web server or
+to any S3-compatible platform. At the moment we only support binary
+data file uploads to the final server.
+
+If the `upload` params is present, `upload[url]` param is required.
+ (**Note:** This feature was introduced in GitLab 10.7)
+
```http
POST /projects/:id/export
```
@@ -16,9 +24,12 @@ POST /projects/:id/export
| --------- | -------------- | -------- | ---------------------------------------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
| `description` | string | no | Overrides the project description |
+| `upload` | hash | no | Hash that contains the information to upload the exported project to a web server |
+| `upload[url]` | string | yes | The URL to upload the project |
+| `upload[http_method]` | string | no | The HTTP method to upload the exported project. Only `PUT` and `POST` methods allowed. Default is `PUT` |
```console
-curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form "description=Foo Bar" https://gitlab.example.com/api/v4/projects/1/export
+curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/1/export --data "description=FooBar&upload[http_method]=PUT&upload[url]=https://example-bucket.s3.eu-west-3.amazonaws.com/backup?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIMBJHN2O62W8IELQ%2F20180312%2Feu-west-3%2Fs3%2Faws4_request&X-Amz-Date=20180312T110328Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host&X-Amz-Signature=8413facb20ff33a49a147a0b4abcff4c8487cc33ee1f7e450c46e8f695569dbd"
```
```json
@@ -43,7 +54,11 @@ GET /projects/:id/export
curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/1/export
```
-Status can be one of `none`, `started`, or `finished`.
+Status can be one of `none`, `started`, `after_export_action` or `finished`. The
+`after_export_action` state represents that the export process has been completed successfully and
+the platform is performing some actions on the resulted file. For example, sending
+an email notifying the user to download the file, uploading the exported file
+to a web server, etc.
`_links` are only present when export has finished.
diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb
index efc4a33ae1b..5ef4e9d530c 100644
--- a/lib/api/project_export.rb
+++ b/lib/api/project_export.rb
@@ -33,11 +33,28 @@ module API
end
params do
optional :description, type: String, desc: 'Override the project description'
+ optional :upload, type: Hash do
+ optional :url, type: String, desc: 'The URL to upload the project'
+ optional :http_method, type: String, default: 'PUT', desc: 'HTTP method to upload the exported project'
+ end
end
post ':id/export' do
project_export_params = declared_params(include_missing: false)
+ after_export_params = project_export_params.delete(:upload) || {}
- user_project.add_export_job(current_user: current_user, params: project_export_params)
+ export_strategy = if after_export_params[:url].present?
+ params = after_export_params.slice(:url, :http_method).symbolize_keys
+
+ Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy.new(params)
+ end
+
+ if export_strategy&.invalid?
+ render_validation_error!(export_strategy)
+ else
+ user_project.add_export_job(current_user: current_user,
+ after_export_strategy: export_strategy,
+ params: project_export_params)
+ end
accepted!
end
diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
new file mode 100644
index 00000000000..aef371d81eb
--- /dev/null
+++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
@@ -0,0 +1,83 @@
+module Gitlab
+ module ImportExport
+ module AfterExportStrategies
+ class BaseAfterExportStrategy
+ include ActiveModel::Validations
+ extend Forwardable
+
+ StrategyError = Class.new(StandardError)
+
+ AFTER_EXPORT_LOCK_FILE_NAME = '.after_export_action'.freeze
+
+ private
+
+ attr_reader :project, :current_user
+
+ public
+
+ def initialize(attributes = {})
+ @options = OpenStruct.new(attributes)
+
+ self.class.instance_eval do
+ def_delegators :@options, *attributes.keys
+ end
+ end
+
+ def execute(current_user, project)
+ return unless project&.export_project_path
+
+ @project = project
+ @current_user = current_user
+
+ if invalid?
+ log_validation_errors
+
+ return
+ end
+
+ create_or_update_after_export_lock
+ strategy_execute
+
+ true
+ rescue => e
+ project.import_export_shared.error(e)
+ false
+ ensure
+ delete_after_export_lock
+ end
+
+ def to_json(options = {})
+ @options.to_h.merge!(klass: self.class.name).to_json
+ end
+
+ def self.lock_file_path(project)
+ return unless project&.export_path
+
+ File.join(project.export_path, AFTER_EXPORT_LOCK_FILE_NAME)
+ end
+
+ protected
+
+ def strategy_execute
+ raise NotImplementedError
+ end
+
+ private
+
+ def create_or_update_after_export_lock
+ FileUtils.touch(self.class.lock_file_path(project))
+ end
+
+ def delete_after_export_lock
+ lock_file = self.class.lock_file_path(project)
+
+ FileUtils.rm(lock_file) if lock_file.present? && File.exist?(lock_file)
+ end
+
+ def log_validation_errors
+ errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
new file mode 100644
index 00000000000..4371a7eff56
--- /dev/null
+++ b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
@@ -0,0 +1,17 @@
+module Gitlab
+ module ImportExport
+ module AfterExportStrategies
+ class DownloadNotificationStrategy < BaseAfterExportStrategy
+ private
+
+ def strategy_execute
+ notification_service.project_exported(project, current_user)
+ end
+
+ def notification_service
+ @notification_service ||= NotificationService.new
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
new file mode 100644
index 00000000000..938664a95a1
--- /dev/null
+++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
@@ -0,0 +1,61 @@
+module Gitlab
+ module ImportExport
+ module AfterExportStrategies
+ class WebUploadStrategy < BaseAfterExportStrategy
+ PUT_METHOD = 'PUT'.freeze
+ POST_METHOD = 'POST'.freeze
+ INVALID_HTTP_METHOD = 'invalid. Only PUT and POST methods allowed.'.freeze
+
+ validates :url, url: true
+
+ validate do
+ unless [PUT_METHOD, POST_METHOD].include?(http_method.upcase)
+ errors.add(:http_method, INVALID_HTTP_METHOD)
+ end
+ end
+
+ def initialize(url:, http_method: PUT_METHOD)
+ super
+ end
+
+ protected
+
+ def strategy_execute
+ handle_response_error(send_file)
+
+ project.remove_exported_project_file
+ end
+
+ def handle_response_error(response)
+ unless response.success?
+ error_code = response.dig('Error', 'Code') || response.code
+ error_message = response.dig('Error', 'Message') || response.message
+
+ raise StrategyError.new("Error uploading the project. Code #{error_code}: #{error_message}")
+ end
+ end
+
+ private
+
+ def send_file
+ export_file = File.open(project.export_project_path)
+
+ Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options(export_file)) # rubocop:disable GitlabSecurity/PublicSend
+ ensure
+ export_file.close if export_file
+ end
+
+ def send_file_options(export_file)
+ {
+ body_stream: export_file,
+ headers: headers
+ }
+ end
+
+ def headers
+ { 'Content-Length' => File.size(project.export_project_path).to_s }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/after_export_strategy_builder.rb b/lib/gitlab/import_export/after_export_strategy_builder.rb
new file mode 100644
index 00000000000..7eabcae2380
--- /dev/null
+++ b/lib/gitlab/import_export/after_export_strategy_builder.rb
@@ -0,0 +1,24 @@
+module Gitlab
+ module ImportExport
+ class AfterExportStrategyBuilder
+ StrategyNotFoundError = Class.new(StandardError)
+
+ def self.build!(strategy_klass, attributes = {})
+ return default_strategy.new unless strategy_klass
+
+ attributes ||= {}
+ klass = strategy_klass.constantize rescue nil
+
+ unless klass && klass < AfterExportStrategies::BaseAfterExportStrategy
+ raise StrategyNotFoundError.new("Strategy #{strategy_klass} not found")
+ end
+
+ klass.new(**attributes.symbolize_keys)
+ end
+
+ def self.default_strategy
+ AfterExportStrategies::DownloadNotificationStrategy
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb
index 3d3d998a6a3..6d7c36ce38b 100644
--- a/lib/gitlab/import_export/shared.rb
+++ b/lib/gitlab/import_export/shared.rb
@@ -22,7 +22,7 @@ module Gitlab
def error(error)
error_out(error.message, caller[0].dup)
- @errors << error.message
+ add_error_message(error.message)
# Debug:
if error.backtrace
@@ -32,6 +32,14 @@ module Gitlab
end
end
+ def add_error_message(error_message)
+ @errors << error_message
+ end
+
+ def after_export_in_progress?
+ File.exist?(after_export_lock_file)
+ end
+
private
def relative_path
@@ -45,6 +53,10 @@ module Gitlab
def error_out(message, caller)
Rails.logger.error("Import/Export error raised on #{caller}: #{message}")
end
+
+ def after_export_lock_file
+ AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project)
+ end
end
end
end
diff --git a/spec/fixtures/api/schemas/public_api/v4/project/export_status.json b/spec/fixtures/api/schemas/public_api/v4/project/export_status.json
index d24a6f93f4b..81c8815caf6 100644
--- a/spec/fixtures/api/schemas/public_api/v4/project/export_status.json
+++ b/spec/fixtures/api/schemas/public_api/v4/project/export_status.json
@@ -1,7 +1,9 @@
{
"type": "object",
"allOf": [
- { "$ref": "identity.json" },
+ {
+ "$ref": "identity.json"
+ },
{
"required": [
"export_status"
@@ -9,7 +11,12 @@
"properties": {
"export_status": {
"type": "string",
- "enum": ["none", "started", "finished"]
+ "enum": [
+ "none",
+ "started",
+ "finished",
+ "after_export_action"
+ ]
}
}
}
diff --git a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
new file mode 100644
index 00000000000..ed54d87de4a
--- /dev/null
+++ b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
@@ -0,0 +1,104 @@
+require 'spec_helper'
+
+describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do
+ let!(:service) { described_class.new }
+ let!(:project) { create(:project, :with_export) }
+ let(:shared) { project.import_export_shared }
+ let!(:user) { create(:user) }
+
+ describe '#execute' do
+ before do
+ allow(service).to receive(:strategy_execute)
+ end
+
+ it 'returns if project exported file is not found' do
+ allow(project).to receive(:export_project_path).and_return(nil)
+
+ expect(service).not_to receive(:strategy_execute)
+
+ service.execute(user, project)
+ end
+
+ it 'creates a lock file in the export dir' do
+ allow(service).to receive(:delete_after_export_lock)
+
+ service.execute(user, project)
+
+ expect(lock_path_exist?).to be_truthy
+ end
+
+ context 'when the method succeeds' do
+ it 'removes the lock file' do
+ service.execute(user, project)
+
+ expect(lock_path_exist?).to be_falsey
+ end
+ end
+
+ context 'when the method fails' do
+ before do
+ allow(service).to receive(:strategy_execute).and_call_original
+ end
+
+ context 'when validation fails' do
+ before do
+ allow(service).to receive(:invalid?).and_return(true)
+ end
+
+ it 'does not create the lock file' do
+ expect(service).not_to receive(:create_or_update_after_export_lock)
+
+ service.execute(user, project)
+ end
+
+ it 'does not execute main logic' do
+ expect(service).not_to receive(:strategy_execute)
+
+ service.execute(user, project)
+ end
+
+ it 'logs validation errors in shared context' do
+ expect(service).to receive(:log_validation_errors)
+
+ service.execute(user, project)
+ end
+ end
+
+ context 'when an exception is raised' do
+ it 'removes the lock' do
+ expect { service.execute(user, project) }.to raise_error(NotImplementedError)
+
+ expect(lock_path_exist?).to be_falsey
+ end
+ end
+ end
+ end
+
+ describe '#log_validation_errors' do
+ it 'add the message to the shared context' do
+ errors = %w(test_message test_message2)
+
+ allow(service).to receive(:invalid?).and_return(true)
+ allow(service.errors).to receive(:full_messages).and_return(errors)
+
+ expect(shared).to receive(:add_error_message).twice.and_call_original
+
+ service.execute(user, project)
+
+ expect(shared.errors).to eq errors
+ end
+ end
+
+ describe '#to_json' do
+ it 'adds the current strategy class to the serialized attributes' do
+ params = { param1: 1 }
+ result = params.merge(klass: described_class.to_s).to_json
+
+ expect(described_class.new(params).to_json).to eq result
+ end
+ end
+
+ def lock_path_exist?
+ File.exist?(described_class.lock_file_path(project))
+ end
+end
diff --git a/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
new file mode 100644
index 00000000000..5fe57d9987b
--- /dev/null
+++ b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do
+ let(:example_url) { 'http://www.example.com' }
+ let(:strategy) { subject.new(url: example_url, http_method: 'post') }
+ let!(:project) { create(:project, :with_export) }
+ let!(:user) { build(:user) }
+
+ subject { described_class }
+
+ describe 'validations' do
+ it 'only POST and PUT method allowed' do
+ %w(POST post PUT put).each do |method|
+ expect(subject.new(url: example_url, http_method: method)).to be_valid
+ end
+
+ expect(subject.new(url: example_url, http_method: 'whatever')).not_to be_valid
+ end
+
+ it 'onyl allow urls as upload urls' do
+ expect(subject.new(url: example_url)).to be_valid
+ expect(subject.new(url: 'whatever')).not_to be_valid
+ end
+ end
+
+ describe '#execute' do
+ it 'removes the exported project file after the upload' do
+ allow(strategy).to receive(:send_file)
+ allow(strategy).to receive(:handle_response_error)
+
+ expect(project).to receive(:remove_exported_project_file)
+
+ strategy.execute(user, project)
+ end
+ end
+end
diff --git a/spec/lib/gitlab/import_export/after_export_strategy_builder_spec.rb b/spec/lib/gitlab/import_export/after_export_strategy_builder_spec.rb
new file mode 100644
index 00000000000..bf727285a9f
--- /dev/null
+++ b/spec/lib/gitlab/import_export/after_export_strategy_builder_spec.rb
@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe Gitlab::ImportExport::AfterExportStrategyBuilder do
+ let!(:strategies_namespace) { 'Gitlab::ImportExport::AfterExportStrategies' }
+
+ describe '.build!' do
+ context 'when klass param is' do
+ it 'null it returns the default strategy' do
+ expect(described_class.build!(nil).class).to eq described_class.default_strategy
+ end
+
+ it 'not a valid class it raises StrategyNotFoundError exception' do
+ expect { described_class.build!('Whatever') }.to raise_error(described_class::StrategyNotFoundError)
+ end
+
+ it 'not a descendant of AfterExportStrategy' do
+ expect { described_class.build!('User') }.to raise_error(described_class::StrategyNotFoundError)
+ end
+ end
+
+ it 'initializes strategy with attributes param' do
+ params = { param1: 1, param2: 2, param3: 3 }
+
+ strategy = described_class.build!("#{strategies_namespace}::DownloadNotificationStrategy", params)
+
+ params.each { |k, v| expect(strategy.public_send(k)).to eq v }
+ end
+ end
+end
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index 05014222623..96adf64bcec 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -2560,7 +2560,7 @@ describe Project do
end
end
- describe '#remove_exports' do
+ describe '#remove_export' do
let(:legacy_project) { create(:project, :legacy_storage, :with_export) }
let(:project) { create(:project, :with_export) }
@@ -2608,6 +2608,23 @@ describe Project do
end
end
+ describe '#remove_exported_project_file' do
+ let(:project) { create(:project, :with_export) }
+
+ it 'removes the exported project file' do
+ exported_file = project.export_project_path
+
+ expect(File.exist?(exported_file)).to be_truthy
+
+ allow(FileUtils).to receive(:rm_f).and_call_original
+ expect(FileUtils).to receive(:rm_f).with(exported_file).and_call_original
+
+ project.remove_exported_project_file
+
+ expect(File.exist?(exported_file)).to be_falsy
+ end
+ end
+
describe '#forks_count' do
it 'returns the number of forks' do
project = build(:project)
diff --git a/spec/requests/api/project_export_spec.rb b/spec/requests/api/project_export_spec.rb
index 12583109b59..3834d27d0a9 100644
--- a/spec/requests/api/project_export_spec.rb
+++ b/spec/requests/api/project_export_spec.rb
@@ -5,6 +5,7 @@ describe API::ProjectExport do
set(:project_none) { create(:project) }
set(:project_started) { create(:project) }
set(:project_finished) { create(:project) }
+ set(:project_after_export) { create(:project) }
set(:user) { create(:user) }
set(:admin) { create(:admin) }
@@ -12,11 +13,13 @@ describe API::ProjectExport do
let(:path_none) { "/projects/#{project_none.id}/export" }
let(:path_started) { "/projects/#{project_started.id}/export" }
let(:path_finished) { "/projects/#{project_finished.id}/export" }
+ let(:path_after_export) { "/projects/#{project_after_export.id}/export" }
let(:download_path) { "/projects/#{project.id}/export/download" }
let(:download_path_none) { "/projects/#{project_none.id}/export/download" }
let(:download_path_started) { "/projects/#{project_started.id}/export/download" }
let(:download_path_finished) { "/projects/#{project_finished.id}/export/download" }
+ let(:download_path_export_action) { "/projects/#{project_after_export.id}/export/download" }
let(:export_path) { "#{Dir.tmpdir}/project_export_spec" }
@@ -29,6 +32,11 @@ describe API::ProjectExport do
# simulate exported
FileUtils.mkdir_p project_finished.export_path
FileUtils.touch File.join(project_finished.export_path, '_export.tar.gz')
+
+ # simulate in after export action
+ FileUtils.mkdir_p project_after_export.export_path
+ FileUtils.touch File.join(project_after_export.export_path, '_export.tar.gz')
+ FileUtils.touch Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project_after_export)
end
after do
@@ -73,6 +81,14 @@ describe API::ProjectExport do
expect(json_response['export_status']).to eq('started')
end
+ it 'is after_export' do
+ get api(path_after_export, user)
+
+ expect(response).to have_gitlab_http_status(200)
+ expect(response).to match_response_schema('public_api/v4/project/export_status')
+ expect(json_response['export_status']).to eq('after_export_action')
+ end
+
it 'is finished' do
get api(path_finished, user)
@@ -99,6 +115,7 @@ describe API::ProjectExport do
project_none.add_master(user)
project_started.add_master(user)
project_finished.add_master(user)
+ project_after_export.add_master(user)
end
it_behaves_like 'get project export status ok'
@@ -163,6 +180,36 @@ describe API::ProjectExport do
end
end
+ shared_examples_for 'get project export upload after action' do
+ context 'and is uploading' do
+ it 'downloads' do
+ get api(download_path_export_action, user)
+
+ expect(response).to have_gitlab_http_status(200)
+ end
+ end
+
+ context 'when upload complete' do
+ before do
+ FileUtils.rm_rf(project_after_export.export_path)
+ end
+
+ it_behaves_like '404 response' do
+ let(:request) { get api(download_path_export_action, user) }
+ end
+ end
+ end
+
+ shared_examples_for 'get project download by strategy' do
+ context 'when upload strategy set' do
+ it_behaves_like 'get project export upload after action'
+ end
+
+ context 'when download strategy set' do
+ it_behaves_like 'get project export download'
+ end
+ end
+
it_behaves_like 'when project export is disabled' do
let(:request) { get api(download_path, admin) }
end
@@ -171,7 +218,7 @@ describe API::ProjectExport do
context 'when user is an admin' do
let(:user) { admin }
- it_behaves_like 'get project export download'
+ it_behaves_like 'get project download by strategy'
end
context 'when user is a master' do
@@ -180,9 +227,10 @@ describe API::ProjectExport do
project_none.add_master(user)
project_started.add_master(user)
project_finished.add_master(user)
+ project_after_export.add_master(user)
end
- it_behaves_like 'get project export download'
+ it_behaves_like 'get project download by strategy'
end
context 'when user is a developer' do
@@ -229,10 +277,30 @@ describe API::ProjectExport do
end
shared_examples_for 'post project export start' do
- it 'starts' do
- post api(path, user)
+ context 'with upload strategy' do
+ context 'when params invalid' do
+ it_behaves_like '400 response' do
+ let(:request) { post(api(path, user), 'upload[url]' => 'whatever') }
+ end
+ end
+
+ it 'starts' do
+ allow_any_instance_of(Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy).to receive(:send_file)
+
+ post(api(path, user), 'upload[url]' => 'http://gitlab.com')
- expect(response).to have_gitlab_http_status(202)
+ expect(response).to have_gitlab_http_status(202)
+ end
+ end
+
+ context 'with download strategy' do
+ it 'starts' do
+ expect_any_instance_of(Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy).not_to receive(:send_file)
+
+ post api(path, user)
+
+ expect(response).to have_gitlab_http_status(202)
+ end
end
end
@@ -253,6 +321,7 @@ describe API::ProjectExport do
project_none.add_master(user)
project_started.add_master(user)
project_finished.add_master(user)
+ project_after_export.add_master(user)
end
it_behaves_like 'post project export start'
diff --git a/spec/services/projects/import_export/export_service_spec.rb b/spec/services/projects/import_export/export_service_spec.rb
new file mode 100644
index 00000000000..51491c7d529
--- /dev/null
+++ b/spec/services/projects/import_export/export_service_spec.rb
@@ -0,0 +1,85 @@
+require 'spec_helper'
+
+describe Projects::ImportExport::ExportService do
+ describe '#execute' do
+ let!(:user) { create(:user) }
+ let(:project) { create(:project) }
+ let(:shared) { project.import_export_shared }
+ let(:service) { described_class.new(project, user) }
+ let!(:after_export_strategy) { Gitlab::ImportExport::AfterExportStrategies::DownloadNotificationStrategy.new }
+
+ context 'when all saver services succeed' do
+ before do
+ allow(service).to receive(:save_services).and_return(true)
+ end
+
+ it 'saves the project in the file system' do
+ expect(Gitlab::ImportExport::Saver).to receive(:save).with(project: project, shared: shared)
+
+ service.execute
+ end
+
+ it 'calls the after export strategy' do
+ expect(after_export_strategy).to receive(:execute)
+
+ service.execute(after_export_strategy)
+ end
+
+ context 'when after export strategy fails' do
+ before do
+ allow(after_export_strategy).to receive(:execute).and_return(false)
+ end
+
+ after do
+ service.execute(after_export_strategy)
+ end
+
+ it 'removes the remaining exported data' do
+ allow(shared).to receive(:export_path).and_return('whatever')
+ allow(FileUtils).to receive(:rm_rf)
+
+ expect(FileUtils).to receive(:rm_rf).with(shared.export_path)
+ end
+
+ it 'notifies the user' do
+ expect_any_instance_of(NotificationService).to receive(:project_not_exported)
+ end
+
+ it 'notifies logger' do
+ allow(Rails.logger).to receive(:error)
+
+ expect(Rails.logger).to receive(:error)
+ end
+ end
+ end
+
+ context 'when saver services fail' do
+ before do
+ allow(service).to receive(:save_services).and_return(false)
+ end
+
+ after do
+ expect { service.execute }.to raise_error(Gitlab::ImportExport::Error)
+ end
+
+ it 'removes the remaining exported data' do
+ allow(shared).to receive(:export_path).and_return('whatever')
+ allow(FileUtils).to receive(:rm_rf)
+
+ expect(FileUtils).to receive(:rm_rf).with(shared.export_path)
+ end
+
+ it 'notifies the user' do
+ expect_any_instance_of(NotificationService).to receive(:project_not_exported)
+ end
+
+ it 'notifies logger' do
+ expect(Rails.logger).to receive(:error)
+ end
+
+ it 'the after export strategy is not called' do
+ expect(service).not_to receive(:execute_after_export_action)
+ end
+ end
+ end
+end
diff --git a/spec/workers/project_export_worker_spec.rb b/spec/workers/project_export_worker_spec.rb
new file mode 100644
index 00000000000..8899969c178
--- /dev/null
+++ b/spec/workers/project_export_worker_spec.rb
@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+describe ProjectExportWorker do
+ let!(:user) { create(:user) }
+ let!(:project) { create(:project) }
+
+ subject { described_class.new }
+
+ describe '#perform' do
+ context 'when it succeeds' do
+ it 'calls the ExportService' do
+ expect_any_instance_of(::Projects::ImportExport::ExportService).to receive(:execute)
+
+ subject.perform(user.id, project.id, { 'klass' => 'Gitlab::ImportExport::AfterExportStrategies::DownloadNotificationStrategy' })
+ end
+ end
+
+ context 'when it fails' do
+ it 'raises an exception when params are invalid' do
+ expect_any_instance_of(::Projects::ImportExport::ExportService).not_to receive(:execute)
+
+ expect { subject.perform(1234, project.id, {}) }.to raise_exception(ActiveRecord::RecordNotFound)
+ expect { subject.perform(user.id, 1234, {}) }.to raise_exception(ActiveRecord::RecordNotFound)
+ expect { subject.perform(user.id, project.id, { 'klass' => 'Whatever' }) }.to raise_exception(Gitlab::ImportExport::AfterExportStrategyBuilder::StrategyNotFoundError)
+ end
+ end
+ end
+end