summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJames Lopez <james@jameslopez.es>2018-08-29 15:41:56 +0200
committerJames Lopez <james@jameslopez.es>2018-09-06 16:52:42 +0200
commita2aa505805478540b3150b2f9093c8658e18597d (patch)
tree7f080afe99e546f743e14c8efa5c6a30c2527554
parenta2ea32dd44cc4a104e404325c73a77151913a946 (diff)
downloadgitlab-ce-a2aa505805478540b3150b2f9093c8658e18597d.tar.gz
Refactor code to remove object storage flag from Import/Export
Updated docs, refactor import/export code Fix AvatarUploader path issue Fix project export upload webhook error
-rw-r--r--app/controllers/projects_controller.rb12
-rw-r--r--app/models/concerns/storage/legacy_namespace.rb4
-rw-r--r--app/models/namespace.rb12
-rw-r--r--app/models/project.rb22
-rw-r--r--app/uploaders/avatar_uploader.rb4
-rw-r--r--changelogs/unreleased/48778-remove-old-storage-logic-from-import-export.yml5
-rw-r--r--doc/administration/raketasks/project_import_export.md7
-rw-r--r--lib/api/project_export.rb6
-rw-r--r--lib/gitlab/import_export.rb4
-rw-r--r--lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb16
-rw-r--r--lib/gitlab/import_export/avatar_restorer.rb2
-rw-r--r--lib/gitlab/import_export/avatar_saver.rb11
-rw-r--r--lib/gitlab/import_export/importer.rb2
-rw-r--r--lib/gitlab/import_export/project_tree_restorer.rb2
-rw-r--r--lib/gitlab/import_export/saver.rb14
-rw-r--r--lib/gitlab/import_export/uploads_manager.rb13
-rw-r--r--lib/gitlab/import_export/uploads_restorer.rb24
-rw-r--r--lib/gitlab/import_export/uploads_saver.rb2
-rw-r--r--lib/gitlab/template_helper.rb17
-rw-r--r--lib/tasks/gitlab/update_templates.rake4
-rw-r--r--spec/controllers/projects_controller_spec.rb30
-rw-r--r--spec/factories/projects.rb16
-rw-r--r--spec/features/projects/import_export/export_file_spec.rb1
-rw-r--r--spec/features/projects/import_export/import_file_object_storage_spec.rb103
-rw-r--r--spec/features/projects/import_export/import_file_spec.rb3
-rw-r--r--spec/features/projects/import_export/namespace_export_file_spec.rb68
-rw-r--r--spec/lib/gitlab/cleanup/project_uploads_spec.rb2
-rw-r--r--spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb105
-rw-r--r--spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb3
-rw-r--r--spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb31
-rw-r--r--spec/lib/gitlab/import_export/avatar_saver_spec.rb5
-rw-r--r--spec/lib/gitlab/import_export/file_importer_object_storage_spec.rb89
-rw-r--r--spec/lib/gitlab/import_export/file_importer_spec.rb12
-rw-r--r--spec/lib/gitlab/import_export/importer_object_storage_spec.rb115
-rw-r--r--spec/lib/gitlab/import_export/importer_spec.rb15
-rw-r--r--spec/lib/gitlab/import_export/saver_spec.rb24
-rw-r--r--spec/lib/gitlab/import_export/uploads_manager_spec.rb43
-rw-r--r--spec/lib/gitlab/import_export/uploads_restorer_spec.rb10
-rw-r--r--spec/lib/gitlab/import_export/uploads_saver_spec.rb1
-rw-r--r--spec/models/namespace_spec.rb32
-rw-r--r--spec/models/project_spec.rb65
-rw-r--r--spec/requests/api/project_export_spec.rb33
-rw-r--r--spec/requests/api/project_import_spec.rb1
-rw-r--r--spec/support/import_export/export_file_helper.rb2
44 files changed, 99 insertions, 893 deletions
diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb
index 0eaf9f94e37..646d8329aff 100644
--- a/app/controllers/projects_controller.rb
+++ b/app/controllers/projects_controller.rb
@@ -191,10 +191,8 @@ class ProjectsController < Projects::ApplicationController
end
def download_export
- if export_project_object_storage?
+ if @project.export_project_object_exists?
send_upload(@project.import_export_upload.export_file)
- elsif export_project_path
- send_file export_project_path, disposition: 'attachment'
else
redirect_to(
edit_project_path(@project, anchor: 'js-export-project'),
@@ -425,12 +423,4 @@ class ProjectsController < Projects::ApplicationController
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440')
end
-
- def export_project_path
- @export_project_path ||= @project.export_project_path
- end
-
- def export_project_object_storage?
- @project.export_project_object_exists?
- end
end
diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb
index 3b745657a9e..9785011720a 100644
--- a/app/models/concerns/storage/legacy_namespace.rb
+++ b/app/models/concerns/storage/legacy_namespace.rb
@@ -25,8 +25,6 @@ module Storage
Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path)
end
- remove_exports!
-
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
@@ -101,8 +99,6 @@ module Storage
end
end
end
-
- remove_exports!
end
def remove_legacy_exports!
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index 0deb44d7916..76920c3c039 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -253,18 +253,6 @@ class Namespace < ActiveRecord::Base
end
end
- # Exports belonging to projects with legacy storage are placed in a common
- # subdirectory of the namespace, so a simple `rm -rf` is sufficient to remove
- # them.
- #
- # Exports of projects using hashed storage are placed in a location defined
- # only by the project ID, so each must be removed individually.
- def remove_exports!
- remove_legacy_exports!
-
- all_projects.with_storage_feature(:repository).find_each(&:remove_exports)
- end
-
def refresh_project_authorizations
owner.refresh_authorized_projects
end
diff --git a/app/models/project.rb b/app/models/project.rb
index 97d9fa355ef..35c93aec20a 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -1733,16 +1733,12 @@ class Project < ActiveRecord::Base
import_export_shared.archive_path
end
- def export_project_path
- Dir.glob("#{export_path}/*export.tar.gz").max_by { |f| File.ctime(f) }
- end
-
def export_status
if export_in_progress?
:started
elsif after_export_in_progress?
:after_export_action
- elsif export_project_path || export_project_object_exists?
+ elsif export_project_object_exists?
:finished
else
:none
@@ -1757,21 +1753,15 @@ class Project < ActiveRecord::Base
import_export_shared.after_export_in_progress?
end
- def remove_exports(path = export_path)
- if path.present?
- FileUtils.rm_rf(path)
- elsif export_project_object_exists?
- import_export_upload.remove_export_file!
- import_export_upload.save
- end
- end
+ def remove_exports
+ return unless export_project_object_exists?
- def remove_exported_project_file
- remove_exports(export_project_path)
+ import_export_upload.remove_export_file!
+ import_export_upload.save
end
def export_project_object_exists?
- Gitlab::ImportExport.object_storage? && import_export_upload&.export_file&.file
+ import_export_upload&.export_file&.file
end
def full_path_slug
diff --git a/app/uploaders/avatar_uploader.rb b/app/uploaders/avatar_uploader.rb
index b29ef57b071..8526bc16390 100644
--- a/app/uploaders/avatar_uploader.rb
+++ b/app/uploaders/avatar_uploader.rb
@@ -18,6 +18,10 @@ class AvatarUploader < GitlabUploader
false
end
+ def absolute_path
+ self.class.absolute_path(model.avatar)
+ end
+
private
def dynamic_segment
diff --git a/changelogs/unreleased/48778-remove-old-storage-logic-from-import-export.yml b/changelogs/unreleased/48778-remove-old-storage-logic-from-import-export.yml
new file mode 100644
index 00000000000..4ddbc118e86
--- /dev/null
+++ b/changelogs/unreleased/48778-remove-old-storage-logic-from-import-export.yml
@@ -0,0 +1,5 @@
+---
+title: Update Import/Export to only use new storage uploaders logic
+merge_request: 21409
+author:
+type: added
diff --git a/doc/administration/raketasks/project_import_export.md b/doc/administration/raketasks/project_import_export.md
index 7bd765a35e0..f43bba0a7a7 100644
--- a/doc/administration/raketasks/project_import_export.md
+++ b/doc/administration/raketasks/project_import_export.md
@@ -9,6 +9,7 @@
> application settings (`/admin/application_settings`) under 'Import sources'.
> - The exports are stored in a temporary [shared directory][tmp] and are deleted
> every 24 hours by a specific worker.
+> - ImportExport can use object storage automatically starting from GitLab 11.3
The GitLab Import/Export version can be checked by using:
@@ -30,12 +31,6 @@ sudo gitlab-rake gitlab:import_export:data
bundle exec rake gitlab:import_export:data RAILS_ENV=production
```
-In order to enable Object Storage on the Export, you can use the [feature flag][feature-flags]:
-
-```
-import_export_object_storage
-```
-
[ce-3050]: https://gitlab.com/gitlab-org/gitlab-ce/issues/3050
[feature-flags]: https://docs.gitlab.com/ee/api/features.html
[tmp]: ../../development/shared_files.md
diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb
index 15c57a2fc02..8954a0c0a4c 100644
--- a/lib/api/project_export.rb
+++ b/lib/api/project_export.rb
@@ -21,11 +21,7 @@ module API
detail 'This feature was introduced in GitLab 10.6.'
end
get ':id/export/download' do
- path = user_project.export_project_path
-
- if path
- present_disk_file!(path, File.basename(path), 'application/gzip')
- elsif user_project.export_project_object_exists?
+ if user_project.export_project_object_exists?
present_carrierwave_file!(user_project.import_export_upload.export_file)
else
render_api_error!('404 Not found or has expired', 404)
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index be3710c5b7f..53fe2f8e436 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -40,10 +40,6 @@ module Gitlab
"#{basename[0..FILENAME_LIMIT]}_export.tar.gz"
end
- def object_storage?
- Feature.enabled?(:import_export_object_storage)
- end
-
def version
VERSION
end
diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
index dce8f89c0ab..064092681f8 100644
--- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
@@ -23,7 +23,7 @@ module Gitlab
def strategy_execute
handle_response_error(send_file)
- project.remove_exported_project_file
+ project.remove_exports
end
def handle_response_error(response)
@@ -40,15 +40,11 @@ module Gitlab
def send_file
Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend
ensure
- export_file.close if export_file && !object_storage?
+ export_file.close if export_file
end
def export_file
- if object_storage?
- project.import_export_upload.export_file.file.open
- else
- File.open(project.export_project_path)
- end
+ project.import_export_upload.export_file.open
end
def send_file_options
@@ -63,11 +59,7 @@ module Gitlab
end
def export_size
- if object_storage?
- project.import_export_upload.export_file.file.size
- else
- File.size(project.export_project_path)
- end
+ project.import_export_upload.export_file.file.size
end
end
end
diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb
index cfa595629f4..ded05f73cf8 100644
--- a/lib/gitlab/import_export/avatar_restorer.rb
+++ b/lib/gitlab/import_export/avatar_restorer.rb
@@ -19,7 +19,7 @@ module Gitlab
private
def avatar_export_file
- @avatar_export_file ||= Dir["#{avatar_export_path}/*"].first
+ @avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].first
end
def avatar_export_path
diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb
index 31ef0490cb3..6ffebf83dd2 100644
--- a/lib/gitlab/import_export/avatar_saver.rb
+++ b/lib/gitlab/import_export/avatar_saver.rb
@@ -1,8 +1,6 @@
module Gitlab
module ImportExport
class AvatarSaver
- include Gitlab::ImportExport::CommandLineUtil
-
def initialize(project:, shared:)
@project = project
@shared = shared
@@ -14,19 +12,12 @@ module Gitlab
Gitlab::ImportExport::UploadsManager.new(
project: @project,
shared: @shared,
- relative_export_path: 'avatar',
- from: avatar_path
+ relative_export_path: 'avatar'
).save
rescue => e
@shared.error(e)
false
end
-
- private
-
- def avatar_path
- @project.avatar.path
- end
end
end
end
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index 4e179f63d8c..72d5b9b830c 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -92,8 +92,6 @@ module Gitlab
end
def remove_import_file
- return unless Gitlab::ImportExport.object_storage?
-
upload = @project.import_export_upload
return unless upload&.import_file&.file
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index f4106e03a57..00ea4b833e2 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -199,7 +199,7 @@ module Gitlab
end
def excluded_keys_for_relation(relation)
- @reader.attributes_finder.find_excluded_keys(relation)
+ reader.attributes_finder.find_excluded_keys(relation)
end
end
end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
index 3cd153a4fd2..a66f32f91d2 100644
--- a/lib/gitlab/import_export/saver.rb
+++ b/lib/gitlab/import_export/saver.rb
@@ -18,7 +18,7 @@ module Gitlab
Rails.logger.info("Saved project export #{archive_file}")
- save_on_object_storage if use_object_storage?
+ save_on_object_storage
else
@shared.error(Gitlab::ImportExport::Error.new(error_message))
false
@@ -27,10 +27,8 @@ module Gitlab
@shared.error(e)
false
ensure
- if use_object_storage?
- remove_archive
- remove_export_path
- end
+ remove_archive
+ remove_export_path
end
private
@@ -59,12 +57,8 @@ module Gitlab
upload.save!
end
- def use_object_storage?
- Gitlab::ImportExport.object_storage?
- end
-
def error_message
- "Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}"
+ "Unable to save #{archive_file} into #{@shared.export_path}."
end
end
end
diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb
index e0d4235e65b..8511319cb1c 100644
--- a/lib/gitlab/import_export/uploads_manager.rb
+++ b/lib/gitlab/import_export/uploads_manager.rb
@@ -5,18 +5,13 @@ module Gitlab
UPLOADS_BATCH_SIZE = 100
- def initialize(project:, shared:, relative_export_path: 'uploads', from: nil)
+ def initialize(project:, shared:, relative_export_path: 'uploads')
@project = project
@shared = shared
@relative_export_path = relative_export_path
- @from = from || default_uploads_path
end
def save
- if File.file?(@from) && @relative_export_path == 'avatar'
- copy_files(@from, File.join(uploads_export_path, @project.avatar.filename))
- end
-
copy_project_uploads
true
@@ -55,17 +50,11 @@ module Gitlab
copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path))
else
- next unless Gitlab::ImportExport.object_storage?
-
download_and_copy(uploader)
end
end
end
- def default_uploads_path
- FileUploader.absolute_base_dir(@project)
- end
-
def uploads_export_path
@uploads_export_path ||= File.join(@shared.export_path, @relative_export_path)
end
diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb
index 25f85936227..b4313ff4cb4 100644
--- a/lib/gitlab/import_export/uploads_restorer.rb
+++ b/lib/gitlab/import_export/uploads_restorer.rb
@@ -2,30 +2,14 @@ module Gitlab
module ImportExport
class UploadsRestorer < UploadsSaver
def restore
- if Gitlab::ImportExport.object_storage?
- Gitlab::ImportExport::UploadsManager.new(
- project: @project,
- shared: @shared
- ).restore
- elsif File.directory?(uploads_export_path)
- copy_files(uploads_export_path, uploads_path)
-
- true
- else
- true # Proceed without uploads
- end
+ Gitlab::ImportExport::UploadsManager.new(
+ project: @project,
+ shared: @shared
+ ).restore
rescue => e
@shared.error(e)
false
end
-
- def uploads_path
- FileUploader.absolute_base_dir(@project)
- end
-
- def uploads_export_path
- @uploads_export_path ||= File.join(@shared.export_path, 'uploads')
- end
end
end
end
diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb
index b3f17af5661..0275f686c5e 100644
--- a/lib/gitlab/import_export/uploads_saver.rb
+++ b/lib/gitlab/import_export/uploads_saver.rb
@@ -1,8 +1,6 @@
module Gitlab
module ImportExport
class UploadsSaver
- include Gitlab::ImportExport::CommandLineUtil
-
def initialize(project:, shared:)
@project = project
@shared = shared
diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb
index f24a01e6cf5..fc498dde723 100644
--- a/lib/gitlab/template_helper.rb
+++ b/lib/gitlab/template_helper.rb
@@ -1,24 +1,9 @@
module Gitlab
module TemplateHelper
- include Gitlab::Utils::StrongMemoize
-
def prepare_template_environment(file)
return unless file
- if Gitlab::ImportExport.object_storage?
- params[:import_export_upload] = ImportExportUpload.new(import_file: file)
- else
- FileUtils.mkdir_p(File.dirname(import_upload_path))
- FileUtils.copy_entry(file.path, import_upload_path)
-
- params[:import_source] = import_upload_path
- end
- end
-
- def import_upload_path
- strong_memoize(:import_upload_path) do
- Gitlab::ImportExport.import_upload_path(filename: tmp_filename)
- end
+ params[:import_export_upload] = ImportExportUpload.new(import_file: file)
end
def tmp_filename
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
index a25f7ce59c7..3c873ba3a20 100644
--- a/lib/tasks/gitlab/update_templates.rake
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -6,6 +6,8 @@ namespace :gitlab do
desc "GitLab | Update project templates"
task :update_project_templates do
+ include Gitlab::ImportExport::CommandLineUtil
+
if Rails.env.production?
puts "This rake task is not meant fo production instances".red
exit(1)
@@ -52,7 +54,7 @@ namespace :gitlab do
end
Projects::ImportExport::ExportService.new(project, admin).execute
- FileUtils.cp(project.export_project_path, template.archive_path)
+ download_or_copy_upload(project.import_export_upload.export_file, template.archive_path)
Projects::DestroyService.new(admin, project).execute
puts "Exported #{template.name}".green
end
diff --git a/spec/controllers/projects_controller_spec.rb b/spec/controllers/projects_controller_spec.rb
index c3a66477b6a..3bc9cbe64c5 100644
--- a/spec/controllers/projects_controller_spec.rb
+++ b/spec/controllers/projects_controller_spec.rb
@@ -803,37 +803,7 @@ describe ProjectsController do
project.add_maintainer(user)
end
- context 'object storage disabled' do
- before do
- stub_feature_flags(import_export_object_storage: false)
- end
-
- context 'when project export is enabled' do
- it 'returns 302' do
- get :download_export, namespace_id: project.namespace, id: project
-
- expect(response).to have_gitlab_http_status(302)
- end
- end
-
- context 'when project export is disabled' do
- before do
- stub_application_setting(project_export_enabled?: false)
- end
-
- it 'returns 404' do
- get :download_export, namespace_id: project.namespace, id: project
-
- expect(response).to have_gitlab_http_status(404)
- end
- end
- end
-
context 'object storage enabled' do
- before do
- stub_feature_flags(import_export_object_storage: true)
- end
-
context 'when project export is enabled' do
it 'returns 302' do
get :download_export, namespace_id: project.namespace, id: project
diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb
index 17e457c04a5..c0d819498e1 100644
--- a/spec/factories/projects.rb
+++ b/spec/factories/projects.rb
@@ -103,27 +103,11 @@ FactoryBot.define do
end
trait :with_export do
- before(:create) do |_project, _evaluator|
- allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { false }
- allow(Feature).to receive(:enabled?).with('import_export_object_storage') { false }
- end
-
after(:create) do |project, _evaluator|
ProjectExportWorker.new.perform(project.creator.id, project.id)
end
end
- trait :with_object_export do
- before(:create) do |_project, _evaluator|
- allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { true }
- allow(Feature).to receive(:enabled?).with('import_export_object_storage') { true }
- end
-
- after(:create) do |project, evaluator|
- ProjectExportWorker.new.perform(project.creator.id, project.id)
- end
- end
-
trait :broken_storage do
after(:create) do |project|
project.update_column(:repository_storage, 'broken')
diff --git a/spec/features/projects/import_export/export_file_spec.rb b/spec/features/projects/import_export/export_file_spec.rb
index eb281cd2122..8a418356541 100644
--- a/spec/features/projects/import_export/export_file_spec.rb
+++ b/spec/features/projects/import_export/export_file_spec.rb
@@ -25,7 +25,6 @@ describe 'Import/Export - project export integration test', :js do
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
- stub_feature_flags(import_export_object_storage: false)
end
after do
diff --git a/spec/features/projects/import_export/import_file_object_storage_spec.rb b/spec/features/projects/import_export/import_file_object_storage_spec.rb
deleted file mode 100644
index 0d364543916..00000000000
--- a/spec/features/projects/import_export/import_file_object_storage_spec.rb
+++ /dev/null
@@ -1,103 +0,0 @@
-require 'spec_helper'
-
-describe 'Import/Export - project import integration test', :js do
- include Select2Helper
-
- let(:user) { create(:user) }
- let(:file) { File.join(Rails.root, 'spec', 'features', 'projects', 'import_export', 'test_project_export.tar.gz') }
- let(:export_path) { "#{Dir.tmpdir}/import_file_spec" }
-
- before do
- stub_feature_flags(import_export_object_storage: true)
- stub_uploads_object_storage(FileUploader)
- allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
- gitlab_sign_in(user)
- end
-
- after do
- FileUtils.rm_rf(export_path, secure: true)
- end
-
- context 'when selecting the namespace' do
- let(:user) { create(:admin) }
- let!(:namespace) { user.namespace }
- let(:project_path) { 'test-project-path' + SecureRandom.hex }
-
- context 'prefilled the path' do
- it 'user imports an exported project successfully' do
- visit new_project_path
-
- select2(namespace.id, from: '#project_namespace_id')
- fill_in :project_path, with: project_path, visible: true
- click_import_project_tab
- click_link 'GitLab export'
-
- expect(page).to have_content('Import an exported GitLab project')
- expect(URI.parse(current_url).query).to eq("namespace_id=#{namespace.id}&path=#{project_path}")
-
- attach_file('file', file)
- click_on 'Import project'
-
- expect(Project.count).to eq(1)
-
- project = Project.last
- expect(project).not_to be_nil
- expect(project.description).to eq("Foo Bar")
- expect(project.issues).not_to be_empty
- expect(project.merge_requests).not_to be_empty
- expect(project_hook_exists?(project)).to be true
- expect(wiki_exists?(project)).to be true
- expect(project.import_state.status).to eq('finished')
- end
- end
-
- context 'path is not prefilled' do
- it 'user imports an exported project successfully' do
- visit new_project_path
- click_import_project_tab
- click_link 'GitLab export'
-
- fill_in :path, with: 'test-project-path', visible: true
- attach_file('file', file)
-
- expect { click_on 'Import project' }.to change { Project.count }.by(1)
-
- project = Project.last
- expect(project).not_to be_nil
- expect(page).to have_content("Project 'test-project-path' is being imported")
- end
- end
- end
-
- it 'invalid project' do
- project = create(:project, namespace: user.namespace)
-
- visit new_project_path
-
- select2(user.namespace.id, from: '#project_namespace_id')
- fill_in :project_path, with: project.name, visible: true
- click_import_project_tab
- click_link 'GitLab export'
- attach_file('file', file)
- click_on 'Import project'
-
- page.within('.flash-container') do
- expect(page).to have_content('Project could not be imported')
- end
- end
-
- def wiki_exists?(project)
- wiki = ProjectWiki.new(project)
- wiki.repository.exists? && !wiki.repository.empty?
- end
-
- def project_hook_exists?(project)
- Gitlab::GitalyClient::StorageSettings.allow_disk_access do
- Gitlab::Git::Hook.new('post-receive', project.repository.raw_repository).exists?
- end
- end
-
- def click_import_project_tab
- find('#import-project-tab').click
- end
-end
diff --git a/spec/features/projects/import_export/import_file_spec.rb b/spec/features/projects/import_export/import_file_spec.rb
index 2d86115de12..2936482a1f7 100644
--- a/spec/features/projects/import_export/import_file_spec.rb
+++ b/spec/features/projects/import_export/import_file_spec.rb
@@ -8,7 +8,7 @@ describe 'Import/Export - project import integration test', :js do
let(:export_path) { "#{Dir.tmpdir}/import_file_spec" }
before do
- stub_feature_flags(import_export_object_storage: false)
+ stub_uploads_object_storage(FileUploader)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
gitlab_sign_in(user)
end
@@ -33,7 +33,6 @@ describe 'Import/Export - project import integration test', :js do
expect(page).to have_content('Import an exported GitLab project')
expect(URI.parse(current_url).query).to eq("namespace_id=#{namespace.id}&path=#{project_path}")
- expect(Gitlab::ImportExport).to receive(:import_upload_path).with(filename: /\A\h{32}\z/).and_call_original
attach_file('file', file)
click_on 'Import project'
diff --git a/spec/features/projects/import_export/namespace_export_file_spec.rb b/spec/features/projects/import_export/namespace_export_file_spec.rb
deleted file mode 100644
index 9bb8a2063b5..00000000000
--- a/spec/features/projects/import_export/namespace_export_file_spec.rb
+++ /dev/null
@@ -1,68 +0,0 @@
-require 'spec_helper'
-
-describe 'Import/Export - Namespace export file cleanup', :js do
- let(:export_path) { Dir.mktmpdir('namespace_export_file_spec') }
-
- before do
- allow(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
- stub_feature_flags(import_export_object_storage: false)
- end
-
- after do
- FileUtils.rm_rf(export_path, secure: true)
- end
-
- shared_examples_for 'handling project exports on namespace change' do
- let!(:old_export_path) { project.export_path }
-
- before do
- sign_in(create(:admin))
-
- setup_export_project
- end
-
- context 'moving the namespace' do
- it 'removes the export file' do
- expect(File).to exist(old_export_path)
-
- project.namespace.update!(path: build(:namespace).path)
-
- expect(File).not_to exist(old_export_path)
- end
- end
-
- context 'deleting the namespace' do
- it 'removes the export file' do
- expect(File).to exist(old_export_path)
-
- project.namespace.destroy
-
- expect(File).not_to exist(old_export_path)
- end
- end
- end
-
- describe 'legacy storage' do
- let(:project) { create(:project, :legacy_storage) }
-
- it_behaves_like 'handling project exports on namespace change'
- end
-
- describe 'hashed storage' do
- let(:project) { create(:project) }
-
- it_behaves_like 'handling project exports on namespace change'
- end
-
- def setup_export_project
- visit edit_project_path(project)
-
- expect(page).to have_content('Export project')
-
- find(:link, 'Export project').send_keys(:return)
-
- visit edit_project_path(project)
-
- expect(page).to have_content('Download export')
- end
-end
diff --git a/spec/lib/gitlab/cleanup/project_uploads_spec.rb b/spec/lib/gitlab/cleanup/project_uploads_spec.rb
index 11e605eece6..bf130b8fabd 100644
--- a/spec/lib/gitlab/cleanup/project_uploads_spec.rb
+++ b/spec/lib/gitlab/cleanup/project_uploads_spec.rb
@@ -132,7 +132,6 @@ describe Gitlab::Cleanup::ProjectUploads do
let!(:path) { File.join(FileUploader.root, orphaned.model.full_path, orphaned.path) }
before do
- stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(File.dirname(path))
@@ -156,7 +155,6 @@ describe Gitlab::Cleanup::ProjectUploads do
let!(:new_path) { File.join(FileUploader.root, '-', 'project-lost-found', 'wrong', orphaned.path) }
before do
- stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(File.dirname(path))
diff --git a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb
deleted file mode 100644
index 5059d68e54b..00000000000
--- a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb
+++ /dev/null
@@ -1,105 +0,0 @@
-require 'spec_helper'
-
-describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do
- let!(:service) { described_class.new }
- let!(:project) { create(:project, :with_object_export) }
- let(:shared) { project.import_export_shared }
- let!(:user) { create(:user) }
-
- describe '#execute' do
- before do
- allow(service).to receive(:strategy_execute)
- stub_feature_flags(import_export_object_storage: true)
- end
-
- it 'returns if project exported file is not found' do
- allow(project).to receive(:export_project_object_exists?).and_return(false)
-
- expect(service).not_to receive(:strategy_execute)
-
- service.execute(user, project)
- end
-
- it 'creates a lock file in the export dir' do
- allow(service).to receive(:delete_after_export_lock)
-
- service.execute(user, project)
-
- expect(lock_path_exist?).to be_truthy
- end
-
- context 'when the method succeeds' do
- it 'removes the lock file' do
- service.execute(user, project)
-
- expect(lock_path_exist?).to be_falsey
- end
- end
-
- context 'when the method fails' do
- before do
- allow(service).to receive(:strategy_execute).and_call_original
- end
-
- context 'when validation fails' do
- before do
- allow(service).to receive(:invalid?).and_return(true)
- end
-
- it 'does not create the lock file' do
- expect(service).not_to receive(:create_or_update_after_export_lock)
-
- service.execute(user, project)
- end
-
- it 'does not execute main logic' do
- expect(service).not_to receive(:strategy_execute)
-
- service.execute(user, project)
- end
-
- it 'logs validation errors in shared context' do
- expect(service).to receive(:log_validation_errors)
-
- service.execute(user, project)
- end
- end
-
- context 'when an exception is raised' do
- it 'removes the lock' do
- expect { service.execute(user, project) }.to raise_error(NotImplementedError)
-
- expect(lock_path_exist?).to be_falsey
- end
- end
- end
- end
-
- describe '#log_validation_errors' do
- it 'add the message to the shared context' do
- errors = %w(test_message test_message2)
-
- allow(service).to receive(:invalid?).and_return(true)
- allow(service.errors).to receive(:full_messages).and_return(errors)
-
- expect(shared).to receive(:add_error_message).twice.and_call_original
-
- service.execute(user, project)
-
- expect(shared.errors).to eq errors
- end
- end
-
- describe '#to_json' do
- it 'adds the current strategy class to the serialized attributes' do
- params = { param1: 1 }
- result = params.merge(klass: described_class.to_s).to_json
-
- expect(described_class.new(params).to_json).to eq result
- end
- end
-
- def lock_path_exist?
- File.exist?(described_class.lock_file_path(project))
- end
-end
diff --git a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
index 566b7f46c87..94a12221333 100644
--- a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
+++ b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
@@ -9,11 +9,10 @@ describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do
describe '#execute' do
before do
allow(service).to receive(:strategy_execute)
- stub_feature_flags(import_export_object_storage: false)
end
it 'returns if project exported file is not found' do
- allow(project).to receive(:export_project_path).and_return(nil)
+ allow(project).to receive(:export_project_object_exists?).and_return(false)
expect(service).not_to receive(:strategy_execute)
diff --git a/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
index 7f2e0a4ee2c..ec17ad8541f 100644
--- a/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
+++ b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
@@ -24,34 +24,13 @@ describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do
end
describe '#execute' do
- context 'without object storage' do
- before do
- stub_feature_flags(import_export_object_storage: false)
- end
-
- it 'removes the exported project file after the upload' do
- allow(strategy).to receive(:send_file)
- allow(strategy).to receive(:handle_response_error)
-
- expect(project).to receive(:remove_exported_project_file)
-
- strategy.execute(user, project)
- end
- end
-
- context 'with object storage' do
- before do
- stub_feature_flags(import_export_object_storage: true)
- end
+ it 'removes the exported project file after the upload' do
+ allow(strategy).to receive(:send_file)
+ allow(strategy).to receive(:handle_response_error)
- it 'removes the exported project file after the upload' do
- allow(strategy).to receive(:send_file)
- allow(strategy).to receive(:handle_response_error)
+ expect(project).to receive(:remove_exports)
- expect(project).to receive(:remove_exported_project_file)
-
- strategy.execute(user, project)
- end
+ strategy.execute(user, project)
end
end
end
diff --git a/spec/lib/gitlab/import_export/avatar_saver_spec.rb b/spec/lib/gitlab/import_export/avatar_saver_spec.rb
index 90e6d653d34..2bd1b9924c6 100644
--- a/spec/lib/gitlab/import_export/avatar_saver_spec.rb
+++ b/spec/lib/gitlab/import_export/avatar_saver_spec.rb
@@ -8,8 +8,7 @@ describe Gitlab::ImportExport::AvatarSaver do
before do
FileUtils.mkdir_p("#{shared.export_path}/avatar/")
- allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
- stub_feature_flags(import_export_object_storage: false)
+ allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:export_path).and_return(export_path)
end
after do
@@ -19,7 +18,7 @@ describe Gitlab::ImportExport::AvatarSaver do
it 'saves a project avatar' do
described_class.new(project: project_with_avatar, shared: shared).save
- expect(File).to exist("#{shared.export_path}/avatar/dk.png")
+ expect(File).to exist(Dir["#{shared.export_path}/avatar/**/dk.png"].first)
end
it 'is fine not to have an avatar' do
diff --git a/spec/lib/gitlab/import_export/file_importer_object_storage_spec.rb b/spec/lib/gitlab/import_export/file_importer_object_storage_spec.rb
deleted file mode 100644
index 287745eb40e..00000000000
--- a/spec/lib/gitlab/import_export/file_importer_object_storage_spec.rb
+++ /dev/null
@@ -1,89 +0,0 @@
-require 'spec_helper'
-
-describe Gitlab::ImportExport::FileImporter do
- let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
- let(:storage_path) { "#{Dir.tmpdir}/file_importer_spec" }
- let(:valid_file) { "#{shared.export_path}/valid.json" }
- let(:symlink_file) { "#{shared.export_path}/invalid.json" }
- let(:hidden_symlink_file) { "#{shared.export_path}/.hidden" }
- let(:subfolder_symlink_file) { "#{shared.export_path}/subfolder/invalid.json" }
- let(:evil_symlink_file) { "#{shared.export_path}/.\nevil" }
-
- before do
- stub_const('Gitlab::ImportExport::FileImporter::MAX_RETRIES', 0)
- stub_feature_flags(import_export_object_storage: true)
- stub_uploads_object_storage(FileUploader)
-
- allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(storage_path)
- allow_any_instance_of(Gitlab::ImportExport::CommandLineUtil).to receive(:untar_zxf).and_return(true)
- allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:relative_archive_path).and_return('test')
- allow(SecureRandom).to receive(:hex).and_return('abcd')
- setup_files
- end
-
- after do
- FileUtils.rm_rf(storage_path)
- end
-
- context 'normal run' do
- before do
- described_class.import(project: build(:project), archive_file: '', shared: shared)
- end
-
- it 'removes symlinks in root folder' do
- expect(File.exist?(symlink_file)).to be false
- end
-
- it 'removes hidden symlinks in root folder' do
- expect(File.exist?(hidden_symlink_file)).to be false
- end
-
- it 'removes evil symlinks in root folder' do
- expect(File.exist?(evil_symlink_file)).to be false
- end
-
- it 'removes symlinks in subfolders' do
- expect(File.exist?(subfolder_symlink_file)).to be false
- end
-
- it 'does not remove a valid file' do
- expect(File.exist?(valid_file)).to be true
- end
-
- it 'creates the file in the right subfolder' do
- expect(shared.export_path).to include('test/abcd')
- end
- end
-
- context 'error' do
- before do
- allow_any_instance_of(described_class).to receive(:wait_for_archived_file).and_raise(StandardError)
- described_class.import(project: build(:project), archive_file: '', shared: shared)
- end
-
- it 'removes symlinks in root folder' do
- expect(File.exist?(symlink_file)).to be false
- end
-
- it 'removes hidden symlinks in root folder' do
- expect(File.exist?(hidden_symlink_file)).to be false
- end
-
- it 'removes symlinks in subfolders' do
- expect(File.exist?(subfolder_symlink_file)).to be false
- end
-
- it 'does not remove a valid file' do
- expect(File.exist?(valid_file)).to be true
- end
- end
-
- def setup_files
- FileUtils.mkdir_p("#{shared.export_path}/subfolder/")
- FileUtils.touch(valid_file)
- FileUtils.ln_s(valid_file, symlink_file)
- FileUtils.ln_s(valid_file, subfolder_symlink_file)
- FileUtils.ln_s(valid_file, hidden_symlink_file)
- FileUtils.ln_s(valid_file, evil_symlink_file)
- end
-end
diff --git a/spec/lib/gitlab/import_export/file_importer_spec.rb b/spec/lib/gitlab/import_export/file_importer_spec.rb
index 78fccdf1dfc..bf34cefe18f 100644
--- a/spec/lib/gitlab/import_export/file_importer_spec.rb
+++ b/spec/lib/gitlab/import_export/file_importer_spec.rb
@@ -2,7 +2,7 @@ require 'spec_helper'
describe Gitlab::ImportExport::FileImporter do
let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
- let(:export_path) { "#{Dir.tmpdir}/file_importer_spec" }
+ let(:storage_path) { "#{Dir.tmpdir}/file_importer_spec" }
let(:valid_file) { "#{shared.export_path}/valid.json" }
let(:symlink_file) { "#{shared.export_path}/invalid.json" }
let(:hidden_symlink_file) { "#{shared.export_path}/.hidden" }
@@ -11,7 +11,9 @@ describe Gitlab::ImportExport::FileImporter do
before do
stub_const('Gitlab::ImportExport::FileImporter::MAX_RETRIES', 0)
- allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+ stub_uploads_object_storage(FileUploader)
+
+ allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(storage_path)
allow_any_instance_of(Gitlab::ImportExport::CommandLineUtil).to receive(:untar_zxf).and_return(true)
allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:relative_archive_path).and_return('test')
allow(SecureRandom).to receive(:hex).and_return('abcd')
@@ -19,12 +21,12 @@ describe Gitlab::ImportExport::FileImporter do
end
after do
- FileUtils.rm_rf(export_path)
+ FileUtils.rm_rf(storage_path)
end
context 'normal run' do
before do
- described_class.import(project: nil, archive_file: '', shared: shared)
+ described_class.import(project: build(:project), archive_file: '', shared: shared)
end
it 'removes symlinks in root folder' do
@@ -55,7 +57,7 @@ describe Gitlab::ImportExport::FileImporter do
context 'error' do
before do
allow_any_instance_of(described_class).to receive(:wait_for_archived_file).and_raise(StandardError)
- described_class.import(project: nil, archive_file: '', shared: shared)
+ described_class.import(project: build(:project), archive_file: '', shared: shared)
end
it 'removes symlinks in root folder' do
diff --git a/spec/lib/gitlab/import_export/importer_object_storage_spec.rb b/spec/lib/gitlab/import_export/importer_object_storage_spec.rb
deleted file mode 100644
index 24a994b3611..00000000000
--- a/spec/lib/gitlab/import_export/importer_object_storage_spec.rb
+++ /dev/null
@@ -1,115 +0,0 @@
-require 'spec_helper'
-
-describe Gitlab::ImportExport::Importer do
- let(:user) { create(:user) }
- let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
- let(:shared) { project.import_export_shared }
- let(:project) { create(:project) }
- let(:import_file) { fixture_file_upload('spec/features/projects/import_export/test_project_export.tar.gz') }
-
- subject(:importer) { described_class.new(project) }
-
- before do
- allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
- allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file)
- stub_feature_flags(import_export_object_storage: true)
- stub_uploads_object_storage(FileUploader)
-
- FileUtils.mkdir_p(shared.export_path)
- ImportExportUpload.create(project: project, import_file: import_file)
- end
-
- after do
- FileUtils.rm_rf(test_path)
- end
-
- describe '#execute' do
- it 'succeeds' do
- importer.execute
-
- expect(shared.errors).to be_empty
- end
-
- it 'extracts the archive' do
- expect(Gitlab::ImportExport::FileImporter).to receive(:import).and_call_original
-
- importer.execute
- end
-
- it 'checks the version' do
- expect(Gitlab::ImportExport::VersionChecker).to receive(:check!).and_call_original
-
- importer.execute
- end
-
- context 'all restores are executed' do
- [
- Gitlab::ImportExport::AvatarRestorer,
- Gitlab::ImportExport::RepoRestorer,
- Gitlab::ImportExport::WikiRestorer,
- Gitlab::ImportExport::UploadsRestorer,
- Gitlab::ImportExport::LfsRestorer,
- Gitlab::ImportExport::StatisticsRestorer
- ].each do |restorer|
- it "calls the #{restorer}" do
- fake_restorer = double(restorer.to_s)
-
- expect(fake_restorer).to receive(:restore).and_return(true).at_least(1)
- expect(restorer).to receive(:new).and_return(fake_restorer).at_least(1)
-
- importer.execute
- end
- end
-
- it 'restores the ProjectTree' do
- expect(Gitlab::ImportExport::ProjectTreeRestorer).to receive(:new).and_call_original
-
- importer.execute
- end
-
- it 'removes the import file' do
- expect(importer).to receive(:remove_import_file).and_call_original
-
- importer.execute
-
- expect(project.import_export_upload.import_file&.file).to be_nil
- end
- end
-
- context 'when project successfully restored' do
- let!(:existing_project) { create(:project, namespace: user.namespace) }
- let(:project) { create(:project, namespace: user.namespace, name: 'whatever', path: 'whatever') }
-
- before do
- restorers = double(:restorers, all?: true)
-
- allow(subject).to receive(:import_file).and_return(true)
- allow(subject).to receive(:check_version!).and_return(true)
- allow(subject).to receive(:restorers).and_return(restorers)
- allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path }))
- end
-
- context 'when import_data' do
- context 'has original_path' do
- it 'overwrites existing project' do
- expect_any_instance_of(::Projects::OverwriteProjectService).to receive(:execute).with(existing_project)
-
- subject.execute
- end
- end
-
- context 'has not original_path' do
- before do
- allow(project).to receive(:import_data).and_return(double(data: {}))
- end
-
- it 'does not call the overwrite service' do
- expect_any_instance_of(::Projects::OverwriteProjectService).not_to receive(:execute).with(existing_project)
-
- subject.execute
- end
- end
- end
- end
- end
-end
diff --git a/spec/lib/gitlab/import_export/importer_spec.rb b/spec/lib/gitlab/import_export/importer_spec.rb
index 8053c48ad6c..11f98d782b1 100644
--- a/spec/lib/gitlab/import_export/importer_spec.rb
+++ b/spec/lib/gitlab/import_export/importer_spec.rb
@@ -4,16 +4,18 @@ describe Gitlab::ImportExport::Importer do
let(:user) { create(:user) }
let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
let(:shared) { project.import_export_shared }
- let(:project) { create(:project, import_source: File.join(test_path, 'test_project_export.tar.gz')) }
+ let(:project) { create(:project) }
+ let(:import_file) { fixture_file_upload('spec/features/projects/import_export/test_project_export.tar.gz') }
subject(:importer) { described_class.new(project) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file)
+ stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(shared.export_path)
- FileUtils.cp(Rails.root.join('spec/features/projects/import_export/test_project_export.tar.gz'), test_path)
+ ImportExportUpload.create(project: project, import_file: import_file)
end
after do
@@ -64,6 +66,14 @@ describe Gitlab::ImportExport::Importer do
importer.execute
end
+ it 'removes the import file' do
+ expect(importer).to receive(:remove_import_file).and_call_original
+
+ importer.execute
+
+ expect(project.import_export_upload.import_file&.file).to be_nil
+ end
+
it 'sets the correct visibility_level when visibility level is a string' do
project.create_or_update_import_data(
data: { override_params: { visibility_level: Gitlab::VisibilityLevel::PRIVATE.to_s } }
@@ -85,7 +95,6 @@ describe Gitlab::ImportExport::Importer do
allow(subject).to receive(:import_file).and_return(true)
allow(subject).to receive(:check_version!).and_return(true)
allow(subject).to receive(:restorers).and_return(restorers)
- allow(restorers).to receive(:all?).and_return(true)
allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path }))
end
diff --git a/spec/lib/gitlab/import_export/saver_spec.rb b/spec/lib/gitlab/import_export/saver_spec.rb
index 02f1a4b81aa..d185ff2dfcc 100644
--- a/spec/lib/gitlab/import_export/saver_spec.rb
+++ b/spec/lib/gitlab/import_export/saver_spec.rb
@@ -18,26 +18,12 @@ describe Gitlab::ImportExport::Saver do
FileUtils.rm_rf(export_path)
end
- context 'local archive' do
- it 'saves the repo to disk' do
- stub_feature_flags(import_export_object_storage: false)
+ it 'saves the repo using object storage' do
+ stub_uploads_object_storage(ImportExportUploader)
- subject.save
+ subject.save
- expect(shared.errors).to be_empty
- expect(Dir.empty?(shared.archive_path)).to be false
- end
- end
-
- context 'object storage' do
- it 'saves the repo using object storage' do
- stub_feature_flags(import_export_object_storage: true)
- stub_uploads_object_storage(ImportExportUploader)
-
- subject.save
-
- expect(ImportExportUpload.find_by(project: project).export_file.url)
- .to match(%r[\/uploads\/-\/system\/import_export_upload\/export_file.*])
- end
+ expect(ImportExportUpload.find_by(project: project).export_file.url)
+ .to match(%r[\/uploads\/-\/system\/import_export_upload\/export_file.*])
end
end
diff --git a/spec/lib/gitlab/import_export/uploads_manager_spec.rb b/spec/lib/gitlab/import_export/uploads_manager_spec.rb
index f799de18cd0..792117e1df1 100644
--- a/spec/lib/gitlab/import_export/uploads_manager_spec.rb
+++ b/spec/lib/gitlab/import_export/uploads_manager_spec.rb
@@ -4,6 +4,7 @@ describe Gitlab::ImportExport::UploadsManager do
let(:shared) { project.import_export_shared }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:project) { create(:project) }
+ let(:upload) { create(:upload, :issuable_upload, :object_storage, model: project) }
let(:exported_file_path) { "#{shared.export_path}/uploads/#{upload.secret}/#{File.basename(upload.path)}" }
subject(:manager) { described_class.new(project: project, shared: shared) }
@@ -69,44 +70,20 @@ describe Gitlab::ImportExport::UploadsManager do
end
end
end
+ end
- context 'using object storage' do
- let!(:upload) { create(:upload, :issuable_upload, :object_storage, model: project) }
-
- before do
- stub_feature_flags(import_export_object_storage: true)
- stub_uploads_object_storage(FileUploader)
- end
-
- it 'saves the file' do
- fake_uri = double
-
- expect(fake_uri).to receive(:open).and_return(StringIO.new('File content'))
- expect(URI).to receive(:parse).and_return(fake_uri)
-
- manager.save
+ describe '#restore' do
+ before do
+ stub_uploads_object_storage(FileUploader)
- expect(File.read(exported_file_path)).to eq('File content')
- end
+ FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038'))
+ FileUtils.touch(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038', "dummy.txt"))
end
- describe '#restore' do
- context 'using object storage' do
- before do
- stub_feature_flags(import_export_object_storage: true)
- stub_uploads_object_storage(FileUploader)
-
- FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038'))
- FileUtils.touch(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038', "dummy.txt"))
- end
+ it 'restores the file' do
+ manager.restore
- it 'restores the file' do
- manager.restore
-
- expect(project.uploads.size).to eq(1)
- expect(project.uploads.first.build_uploader.filename).to eq('dummy.txt')
- end
- end
+ expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt')
end
end
end
diff --git a/spec/lib/gitlab/import_export/uploads_restorer_spec.rb b/spec/lib/gitlab/import_export/uploads_restorer_spec.rb
index acef97459b8..6072f18b8c7 100644
--- a/spec/lib/gitlab/import_export/uploads_restorer_spec.rb
+++ b/spec/lib/gitlab/import_export/uploads_restorer_spec.rb
@@ -8,7 +8,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/random'))
- FileUtils.touch(File.join(shared.export_path, 'uploads/random', "dummy.txt"))
+ FileUtils.touch(File.join(shared.export_path, 'uploads/random', 'dummy.txt'))
end
after do
@@ -27,9 +27,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
it 'copies the uploads to the project path' do
subject.restore
- uploads = Dir.glob(File.join(subject.uploads_path, '**/*')).map { |file| File.basename(file) }
-
- expect(uploads).to include('dummy.txt')
+ expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt')
end
end
@@ -45,9 +43,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
it 'copies the uploads to the project path' do
subject.restore
- uploads = Dir.glob(File.join(subject.uploads_path, '**/*')).map { |file| File.basename(file) }
-
- expect(uploads).to include('dummy.txt')
+ expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt')
end
end
end
diff --git a/spec/lib/gitlab/import_export/uploads_saver_spec.rb b/spec/lib/gitlab/import_export/uploads_saver_spec.rb
index c716edd9397..24993460e51 100644
--- a/spec/lib/gitlab/import_export/uploads_saver_spec.rb
+++ b/spec/lib/gitlab/import_export/uploads_saver_spec.rb
@@ -7,7 +7,6 @@ describe Gitlab::ImportExport::UploadsSaver do
let(:shared) { project.import_export_shared }
before do
- stub_feature_flags(import_export_object_storage: false)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
end
diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb
index 9b7f932ec3a..3649990670b 100644
--- a/spec/models/namespace_spec.rb
+++ b/spec/models/namespace_spec.rb
@@ -394,12 +394,6 @@ describe Namespace do
child.destroy
end
end
-
- it 'removes the exports folder' do
- expect(namespace).to receive(:remove_exports!)
-
- namespace.destroy
- end
end
context 'hashed storage' do
@@ -414,12 +408,6 @@ describe Namespace do
expect(File.exist?(deleted_path_in_dir)).to be(false)
end
-
- it 'removes the exports folder' do
- expect(namespace).to receive(:remove_exports!)
-
- namespace.destroy
- end
end
end
@@ -706,26 +694,6 @@ describe Namespace do
end
end
- describe '#remove_exports' do
- let(:legacy_project) { create(:project, :with_export, :legacy_storage, namespace: namespace) }
- let(:hashed_project) { create(:project, :with_export, namespace: namespace) }
- let(:export_path) { Dir.mktmpdir('namespace_remove_exports_spec') }
- let(:legacy_export) { legacy_project.export_project_path }
- let(:hashed_export) { hashed_project.export_project_path }
-
- it 'removes exports for legacy and hashed projects' do
- allow(Gitlab::ImportExport).to receive(:storage_path) { export_path }
-
- expect(File.exist?(legacy_export)).to be_truthy
- expect(File.exist?(hashed_export)).to be_truthy
-
- namespace.remove_exports!
-
- expect(File.exist?(legacy_export)).to be_falsy
- expect(File.exist?(hashed_export)).to be_falsy
- end
- end
-
describe '#full_path_was' do
context 'when the group has no parent' do
it 'should return the path was' do
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index 264632dba4b..809a21e80ee 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -2854,73 +2854,12 @@ describe Project do
end
describe '#remove_export' do
- let(:legacy_project) { create(:project, :legacy_storage, :with_export) }
let(:project) { create(:project, :with_export) }
- before do
- stub_feature_flags(import_export_object_storage: false)
- end
-
- it 'removes the exports directory for the project' do
- expect(File.exist?(project.export_path)).to be_truthy
-
- allow(FileUtils).to receive(:rm_rf).and_call_original
- expect(FileUtils).to receive(:rm_rf).with(project.export_path).and_call_original
+ it 'removes the export' do
project.remove_exports
- expect(File.exist?(project.export_path)).to be_falsy
- end
-
- it 'is a no-op on legacy projects when there is no namespace' do
- export_path = legacy_project.export_path
-
- legacy_project.namespace.delete
- legacy_project.reload
-
- expect(FileUtils).not_to receive(:rm_rf).with(export_path)
-
- legacy_project.remove_exports
-
- expect(File.exist?(export_path)).to be_truthy
- end
-
- it 'runs on hashed storage projects when there is no namespace' do
- export_path = project.export_path
-
- project.namespace.delete
- legacy_project.reload
-
- allow(FileUtils).to receive(:rm_rf).and_call_original
- expect(FileUtils).to receive(:rm_rf).with(export_path).and_call_original
-
- project.remove_exports
-
- expect(File.exist?(export_path)).to be_falsy
- end
-
- it 'is run when the project is destroyed' do
- expect(project).to receive(:remove_exports).and_call_original
-
- project.destroy
- end
- end
-
- describe '#remove_exported_project_file' do
- let(:project) { create(:project, :with_export) }
-
- it 'removes the exported project file' do
- stub_feature_flags(import_export_object_storage: false)
-
- exported_file = project.export_project_path
-
- expect(File.exist?(exported_file)).to be_truthy
-
- allow(FileUtils).to receive(:rm_rf).and_call_original
- expect(FileUtils).to receive(:rm_rf).with(exported_file).and_call_original
-
- project.remove_exported_project_file
-
- expect(File.exist?(exported_file)).to be_falsy
+ expect(project.export_project_object_exists?).to be_falsey
end
end
diff --git a/spec/requests/api/project_export_spec.rb b/spec/requests/api/project_export_spec.rb
index 45e4e35d773..0d4854a83b9 100644
--- a/spec/requests/api/project_export_spec.rb
+++ b/spec/requests/api/project_export_spec.rb
@@ -4,8 +4,8 @@ describe API::ProjectExport do
set(:project) { create(:project) }
set(:project_none) { create(:project) }
set(:project_started) { create(:project) }
- set(:project_finished) { create(:project) }
- set(:project_after_export) { create(:project) }
+ let(:project_finished) { create(:project, :with_export) }
+ let(:project_after_export) { create(:project, :with_export) }
set(:user) { create(:user) }
set(:admin) { create(:admin) }
@@ -29,13 +29,7 @@ describe API::ProjectExport do
# simulate exporting work directory
FileUtils.mkdir_p File.join(project_started.export_path, 'securerandom-hex')
- # simulate exported
- FileUtils.mkdir_p project_finished.export_path
- FileUtils.touch File.join(project_finished.export_path, '_export.tar.gz')
-
# simulate in after export action
- FileUtils.mkdir_p project_after_export.export_path
- FileUtils.touch File.join(project_after_export.export_path, '_export.tar.gz')
FileUtils.touch Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project_after_export)
end
@@ -191,14 +185,11 @@ describe API::ProjectExport do
context 'when upload complete' do
before do
- FileUtils.rm_rf(project_after_export.export_path)
-
- if project_after_export.export_project_object_exists?
- upload = project_after_export.import_export_upload
+ project_after_export.remove_exports
+ end
- upload.remove_export_file!
- upload.save
- end
+ it 'has removed the export' do
+ expect(project_after_export.export_project_object_exists?).to be_falsey
end
it_behaves_like '404 response' do
@@ -273,13 +264,13 @@ describe API::ProjectExport do
before do
stub_uploads_object_storage(ImportExportUploader)
- [project, project_finished, project_after_export].each do |p|
- p.add_maintainer(user)
+ project.add_maintainer(user)
+ project_finished.add_maintainer(user)
+ project_after_export.add_maintainer(user)
- upload = ImportExportUpload.new(project: p)
- upload.export_file = fixture_file_upload('spec/fixtures/project_export.tar.gz', "`/tar.gz")
- upload.save!
- end
+ upload = ImportExportUpload.new(project: project)
+ upload.export_file = fixture_file_upload('spec/fixtures/project_export.tar.gz', "`/tar.gz")
+ upload.save!
end
it_behaves_like 'get project download by strategy'
diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb
index bc06f3c3732..c8fa4754810 100644
--- a/spec/requests/api/project_import_spec.rb
+++ b/spec/requests/api/project_import_spec.rb
@@ -7,7 +7,6 @@ describe API::ProjectImport do
let(:namespace) { create(:group) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
- stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
namespace.add_owner(user)
diff --git a/spec/support/import_export/export_file_helper.rb b/spec/support/import_export/export_file_helper.rb
index 4d925ac77f4..cbe238e20ea 100644
--- a/spec/support/import_export/export_file_helper.rb
+++ b/spec/support/import_export/export_file_helper.rb
@@ -52,7 +52,7 @@ module ExportFileHelper
# Expands the compressed file for an exported project into +tmpdir+
def in_directory_with_expanded_export(project)
Dir.mktmpdir do |tmpdir|
- export_file = project.export_project_path
+ export_file = project.import_export_upload.export_file.path
_output, exit_status = Gitlab::Popen.popen(%W{tar -zxf #{export_file} -C #{tmpdir}})
yield(exit_status, tmpdir)