summaryrefslogtreecommitdiff
path: root/spec
diff options
context:
space:
mode:
authorJames Lopez <james@jameslopez.es>2018-08-29 15:41:56 +0200
committerJames Lopez <james@jameslopez.es>2018-09-06 16:52:42 +0200
commita2aa505805478540b3150b2f9093c8658e18597d (patch)
tree7f080afe99e546f743e14c8efa5c6a30c2527554 /spec
parenta2ea32dd44cc4a104e404325c73a77151913a946 (diff)
downloadgitlab-ce-a2aa505805478540b3150b2f9093c8658e18597d.tar.gz
Refactor code to remove object storage flag from Import/Export
Updated docs, refactor import/export code Fix AvatarUploader path issue Fix project export upload webhook error
Diffstat (limited to 'spec')
-rw-r--r--spec/controllers/projects_controller_spec.rb30
-rw-r--r--spec/factories/projects.rb16
-rw-r--r--spec/features/projects/import_export/export_file_spec.rb1
-rw-r--r--spec/features/projects/import_export/import_file_object_storage_spec.rb103
-rw-r--r--spec/features/projects/import_export/import_file_spec.rb3
-rw-r--r--spec/features/projects/import_export/namespace_export_file_spec.rb68
-rw-r--r--spec/lib/gitlab/cleanup/project_uploads_spec.rb2
-rw-r--r--spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb105
-rw-r--r--spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb3
-rw-r--r--spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb31
-rw-r--r--spec/lib/gitlab/import_export/avatar_saver_spec.rb5
-rw-r--r--spec/lib/gitlab/import_export/file_importer_object_storage_spec.rb89
-rw-r--r--spec/lib/gitlab/import_export/file_importer_spec.rb12
-rw-r--r--spec/lib/gitlab/import_export/importer_object_storage_spec.rb115
-rw-r--r--spec/lib/gitlab/import_export/importer_spec.rb15
-rw-r--r--spec/lib/gitlab/import_export/saver_spec.rb24
-rw-r--r--spec/lib/gitlab/import_export/uploads_manager_spec.rb43
-rw-r--r--spec/lib/gitlab/import_export/uploads_restorer_spec.rb10
-rw-r--r--spec/lib/gitlab/import_export/uploads_saver_spec.rb1
-rw-r--r--spec/models/namespace_spec.rb32
-rw-r--r--spec/models/project_spec.rb65
-rw-r--r--spec/requests/api/project_export_spec.rb33
-rw-r--r--spec/requests/api/project_import_spec.rb1
-rw-r--r--spec/support/import_export/export_file_helper.rb2
24 files changed, 61 insertions, 748 deletions
diff --git a/spec/controllers/projects_controller_spec.rb b/spec/controllers/projects_controller_spec.rb
index c3a66477b6a..3bc9cbe64c5 100644
--- a/spec/controllers/projects_controller_spec.rb
+++ b/spec/controllers/projects_controller_spec.rb
@@ -803,37 +803,7 @@ describe ProjectsController do
project.add_maintainer(user)
end
- context 'object storage disabled' do
- before do
- stub_feature_flags(import_export_object_storage: false)
- end
-
- context 'when project export is enabled' do
- it 'returns 302' do
- get :download_export, namespace_id: project.namespace, id: project
-
- expect(response).to have_gitlab_http_status(302)
- end
- end
-
- context 'when project export is disabled' do
- before do
- stub_application_setting(project_export_enabled?: false)
- end
-
- it 'returns 404' do
- get :download_export, namespace_id: project.namespace, id: project
-
- expect(response).to have_gitlab_http_status(404)
- end
- end
- end
-
context 'object storage enabled' do
- before do
- stub_feature_flags(import_export_object_storage: true)
- end
-
context 'when project export is enabled' do
it 'returns 302' do
get :download_export, namespace_id: project.namespace, id: project
diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb
index 17e457c04a5..c0d819498e1 100644
--- a/spec/factories/projects.rb
+++ b/spec/factories/projects.rb
@@ -103,27 +103,11 @@ FactoryBot.define do
end
trait :with_export do
- before(:create) do |_project, _evaluator|
- allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { false }
- allow(Feature).to receive(:enabled?).with('import_export_object_storage') { false }
- end
-
after(:create) do |project, _evaluator|
ProjectExportWorker.new.perform(project.creator.id, project.id)
end
end
- trait :with_object_export do
- before(:create) do |_project, _evaluator|
- allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { true }
- allow(Feature).to receive(:enabled?).with('import_export_object_storage') { true }
- end
-
- after(:create) do |project, evaluator|
- ProjectExportWorker.new.perform(project.creator.id, project.id)
- end
- end
-
trait :broken_storage do
after(:create) do |project|
project.update_column(:repository_storage, 'broken')
diff --git a/spec/features/projects/import_export/export_file_spec.rb b/spec/features/projects/import_export/export_file_spec.rb
index eb281cd2122..8a418356541 100644
--- a/spec/features/projects/import_export/export_file_spec.rb
+++ b/spec/features/projects/import_export/export_file_spec.rb
@@ -25,7 +25,6 @@ describe 'Import/Export - project export integration test', :js do
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
- stub_feature_flags(import_export_object_storage: false)
end
after do
diff --git a/spec/features/projects/import_export/import_file_object_storage_spec.rb b/spec/features/projects/import_export/import_file_object_storage_spec.rb
deleted file mode 100644
index 0d364543916..00000000000
--- a/spec/features/projects/import_export/import_file_object_storage_spec.rb
+++ /dev/null
@@ -1,103 +0,0 @@
-require 'spec_helper'
-
-describe 'Import/Export - project import integration test', :js do
- include Select2Helper
-
- let(:user) { create(:user) }
- let(:file) { File.join(Rails.root, 'spec', 'features', 'projects', 'import_export', 'test_project_export.tar.gz') }
- let(:export_path) { "#{Dir.tmpdir}/import_file_spec" }
-
- before do
- stub_feature_flags(import_export_object_storage: true)
- stub_uploads_object_storage(FileUploader)
- allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
- gitlab_sign_in(user)
- end
-
- after do
- FileUtils.rm_rf(export_path, secure: true)
- end
-
- context 'when selecting the namespace' do
- let(:user) { create(:admin) }
- let!(:namespace) { user.namespace }
- let(:project_path) { 'test-project-path' + SecureRandom.hex }
-
- context 'prefilled the path' do
- it 'user imports an exported project successfully' do
- visit new_project_path
-
- select2(namespace.id, from: '#project_namespace_id')
- fill_in :project_path, with: project_path, visible: true
- click_import_project_tab
- click_link 'GitLab export'
-
- expect(page).to have_content('Import an exported GitLab project')
- expect(URI.parse(current_url).query).to eq("namespace_id=#{namespace.id}&path=#{project_path}")
-
- attach_file('file', file)
- click_on 'Import project'
-
- expect(Project.count).to eq(1)
-
- project = Project.last
- expect(project).not_to be_nil
- expect(project.description).to eq("Foo Bar")
- expect(project.issues).not_to be_empty
- expect(project.merge_requests).not_to be_empty
- expect(project_hook_exists?(project)).to be true
- expect(wiki_exists?(project)).to be true
- expect(project.import_state.status).to eq('finished')
- end
- end
-
- context 'path is not prefilled' do
- it 'user imports an exported project successfully' do
- visit new_project_path
- click_import_project_tab
- click_link 'GitLab export'
-
- fill_in :path, with: 'test-project-path', visible: true
- attach_file('file', file)
-
- expect { click_on 'Import project' }.to change { Project.count }.by(1)
-
- project = Project.last
- expect(project).not_to be_nil
- expect(page).to have_content("Project 'test-project-path' is being imported")
- end
- end
- end
-
- it 'invalid project' do
- project = create(:project, namespace: user.namespace)
-
- visit new_project_path
-
- select2(user.namespace.id, from: '#project_namespace_id')
- fill_in :project_path, with: project.name, visible: true
- click_import_project_tab
- click_link 'GitLab export'
- attach_file('file', file)
- click_on 'Import project'
-
- page.within('.flash-container') do
- expect(page).to have_content('Project could not be imported')
- end
- end
-
- def wiki_exists?(project)
- wiki = ProjectWiki.new(project)
- wiki.repository.exists? && !wiki.repository.empty?
- end
-
- def project_hook_exists?(project)
- Gitlab::GitalyClient::StorageSettings.allow_disk_access do
- Gitlab::Git::Hook.new('post-receive', project.repository.raw_repository).exists?
- end
- end
-
- def click_import_project_tab
- find('#import-project-tab').click
- end
-end
diff --git a/spec/features/projects/import_export/import_file_spec.rb b/spec/features/projects/import_export/import_file_spec.rb
index 2d86115de12..2936482a1f7 100644
--- a/spec/features/projects/import_export/import_file_spec.rb
+++ b/spec/features/projects/import_export/import_file_spec.rb
@@ -8,7 +8,7 @@ describe 'Import/Export - project import integration test', :js do
let(:export_path) { "#{Dir.tmpdir}/import_file_spec" }
before do
- stub_feature_flags(import_export_object_storage: false)
+ stub_uploads_object_storage(FileUploader)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
gitlab_sign_in(user)
end
@@ -33,7 +33,6 @@ describe 'Import/Export - project import integration test', :js do
expect(page).to have_content('Import an exported GitLab project')
expect(URI.parse(current_url).query).to eq("namespace_id=#{namespace.id}&path=#{project_path}")
- expect(Gitlab::ImportExport).to receive(:import_upload_path).with(filename: /\A\h{32}\z/).and_call_original
attach_file('file', file)
click_on 'Import project'
diff --git a/spec/features/projects/import_export/namespace_export_file_spec.rb b/spec/features/projects/import_export/namespace_export_file_spec.rb
deleted file mode 100644
index 9bb8a2063b5..00000000000
--- a/spec/features/projects/import_export/namespace_export_file_spec.rb
+++ /dev/null
@@ -1,68 +0,0 @@
-require 'spec_helper'
-
-describe 'Import/Export - Namespace export file cleanup', :js do
- let(:export_path) { Dir.mktmpdir('namespace_export_file_spec') }
-
- before do
- allow(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
- stub_feature_flags(import_export_object_storage: false)
- end
-
- after do
- FileUtils.rm_rf(export_path, secure: true)
- end
-
- shared_examples_for 'handling project exports on namespace change' do
- let!(:old_export_path) { project.export_path }
-
- before do
- sign_in(create(:admin))
-
- setup_export_project
- end
-
- context 'moving the namespace' do
- it 'removes the export file' do
- expect(File).to exist(old_export_path)
-
- project.namespace.update!(path: build(:namespace).path)
-
- expect(File).not_to exist(old_export_path)
- end
- end
-
- context 'deleting the namespace' do
- it 'removes the export file' do
- expect(File).to exist(old_export_path)
-
- project.namespace.destroy
-
- expect(File).not_to exist(old_export_path)
- end
- end
- end
-
- describe 'legacy storage' do
- let(:project) { create(:project, :legacy_storage) }
-
- it_behaves_like 'handling project exports on namespace change'
- end
-
- describe 'hashed storage' do
- let(:project) { create(:project) }
-
- it_behaves_like 'handling project exports on namespace change'
- end
-
- def setup_export_project
- visit edit_project_path(project)
-
- expect(page).to have_content('Export project')
-
- find(:link, 'Export project').send_keys(:return)
-
- visit edit_project_path(project)
-
- expect(page).to have_content('Download export')
- end
-end
diff --git a/spec/lib/gitlab/cleanup/project_uploads_spec.rb b/spec/lib/gitlab/cleanup/project_uploads_spec.rb
index 11e605eece6..bf130b8fabd 100644
--- a/spec/lib/gitlab/cleanup/project_uploads_spec.rb
+++ b/spec/lib/gitlab/cleanup/project_uploads_spec.rb
@@ -132,7 +132,6 @@ describe Gitlab::Cleanup::ProjectUploads do
let!(:path) { File.join(FileUploader.root, orphaned.model.full_path, orphaned.path) }
before do
- stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(File.dirname(path))
@@ -156,7 +155,6 @@ describe Gitlab::Cleanup::ProjectUploads do
let!(:new_path) { File.join(FileUploader.root, '-', 'project-lost-found', 'wrong', orphaned.path) }
before do
- stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(File.dirname(path))
diff --git a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb
deleted file mode 100644
index 5059d68e54b..00000000000
--- a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb
+++ /dev/null
@@ -1,105 +0,0 @@
-require 'spec_helper'
-
-describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do
- let!(:service) { described_class.new }
- let!(:project) { create(:project, :with_object_export) }
- let(:shared) { project.import_export_shared }
- let!(:user) { create(:user) }
-
- describe '#execute' do
- before do
- allow(service).to receive(:strategy_execute)
- stub_feature_flags(import_export_object_storage: true)
- end
-
- it 'returns if project exported file is not found' do
- allow(project).to receive(:export_project_object_exists?).and_return(false)
-
- expect(service).not_to receive(:strategy_execute)
-
- service.execute(user, project)
- end
-
- it 'creates a lock file in the export dir' do
- allow(service).to receive(:delete_after_export_lock)
-
- service.execute(user, project)
-
- expect(lock_path_exist?).to be_truthy
- end
-
- context 'when the method succeeds' do
- it 'removes the lock file' do
- service.execute(user, project)
-
- expect(lock_path_exist?).to be_falsey
- end
- end
-
- context 'when the method fails' do
- before do
- allow(service).to receive(:strategy_execute).and_call_original
- end
-
- context 'when validation fails' do
- before do
- allow(service).to receive(:invalid?).and_return(true)
- end
-
- it 'does not create the lock file' do
- expect(service).not_to receive(:create_or_update_after_export_lock)
-
- service.execute(user, project)
- end
-
- it 'does not execute main logic' do
- expect(service).not_to receive(:strategy_execute)
-
- service.execute(user, project)
- end
-
- it 'logs validation errors in shared context' do
- expect(service).to receive(:log_validation_errors)
-
- service.execute(user, project)
- end
- end
-
- context 'when an exception is raised' do
- it 'removes the lock' do
- expect { service.execute(user, project) }.to raise_error(NotImplementedError)
-
- expect(lock_path_exist?).to be_falsey
- end
- end
- end
- end
-
- describe '#log_validation_errors' do
- it 'add the message to the shared context' do
- errors = %w(test_message test_message2)
-
- allow(service).to receive(:invalid?).and_return(true)
- allow(service.errors).to receive(:full_messages).and_return(errors)
-
- expect(shared).to receive(:add_error_message).twice.and_call_original
-
- service.execute(user, project)
-
- expect(shared.errors).to eq errors
- end
- end
-
- describe '#to_json' do
- it 'adds the current strategy class to the serialized attributes' do
- params = { param1: 1 }
- result = params.merge(klass: described_class.to_s).to_json
-
- expect(described_class.new(params).to_json).to eq result
- end
- end
-
- def lock_path_exist?
- File.exist?(described_class.lock_file_path(project))
- end
-end
diff --git a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
index 566b7f46c87..94a12221333 100644
--- a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
+++ b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
@@ -9,11 +9,10 @@ describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do
describe '#execute' do
before do
allow(service).to receive(:strategy_execute)
- stub_feature_flags(import_export_object_storage: false)
end
it 'returns if project exported file is not found' do
- allow(project).to receive(:export_project_path).and_return(nil)
+ allow(project).to receive(:export_project_object_exists?).and_return(false)
expect(service).not_to receive(:strategy_execute)
diff --git a/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
index 7f2e0a4ee2c..ec17ad8541f 100644
--- a/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
+++ b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
@@ -24,34 +24,13 @@ describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do
end
describe '#execute' do
- context 'without object storage' do
- before do
- stub_feature_flags(import_export_object_storage: false)
- end
-
- it 'removes the exported project file after the upload' do
- allow(strategy).to receive(:send_file)
- allow(strategy).to receive(:handle_response_error)
-
- expect(project).to receive(:remove_exported_project_file)
-
- strategy.execute(user, project)
- end
- end
-
- context 'with object storage' do
- before do
- stub_feature_flags(import_export_object_storage: true)
- end
+ it 'removes the exported project file after the upload' do
+ allow(strategy).to receive(:send_file)
+ allow(strategy).to receive(:handle_response_error)
- it 'removes the exported project file after the upload' do
- allow(strategy).to receive(:send_file)
- allow(strategy).to receive(:handle_response_error)
+ expect(project).to receive(:remove_exports)
- expect(project).to receive(:remove_exported_project_file)
-
- strategy.execute(user, project)
- end
+ strategy.execute(user, project)
end
end
end
diff --git a/spec/lib/gitlab/import_export/avatar_saver_spec.rb b/spec/lib/gitlab/import_export/avatar_saver_spec.rb
index 90e6d653d34..2bd1b9924c6 100644
--- a/spec/lib/gitlab/import_export/avatar_saver_spec.rb
+++ b/spec/lib/gitlab/import_export/avatar_saver_spec.rb
@@ -8,8 +8,7 @@ describe Gitlab::ImportExport::AvatarSaver do
before do
FileUtils.mkdir_p("#{shared.export_path}/avatar/")
- allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
- stub_feature_flags(import_export_object_storage: false)
+ allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:export_path).and_return(export_path)
end
after do
@@ -19,7 +18,7 @@ describe Gitlab::ImportExport::AvatarSaver do
it 'saves a project avatar' do
described_class.new(project: project_with_avatar, shared: shared).save
- expect(File).to exist("#{shared.export_path}/avatar/dk.png")
+ expect(File).to exist(Dir["#{shared.export_path}/avatar/**/dk.png"].first)
end
it 'is fine not to have an avatar' do
diff --git a/spec/lib/gitlab/import_export/file_importer_object_storage_spec.rb b/spec/lib/gitlab/import_export/file_importer_object_storage_spec.rb
deleted file mode 100644
index 287745eb40e..00000000000
--- a/spec/lib/gitlab/import_export/file_importer_object_storage_spec.rb
+++ /dev/null
@@ -1,89 +0,0 @@
-require 'spec_helper'
-
-describe Gitlab::ImportExport::FileImporter do
- let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
- let(:storage_path) { "#{Dir.tmpdir}/file_importer_spec" }
- let(:valid_file) { "#{shared.export_path}/valid.json" }
- let(:symlink_file) { "#{shared.export_path}/invalid.json" }
- let(:hidden_symlink_file) { "#{shared.export_path}/.hidden" }
- let(:subfolder_symlink_file) { "#{shared.export_path}/subfolder/invalid.json" }
- let(:evil_symlink_file) { "#{shared.export_path}/.\nevil" }
-
- before do
- stub_const('Gitlab::ImportExport::FileImporter::MAX_RETRIES', 0)
- stub_feature_flags(import_export_object_storage: true)
- stub_uploads_object_storage(FileUploader)
-
- allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(storage_path)
- allow_any_instance_of(Gitlab::ImportExport::CommandLineUtil).to receive(:untar_zxf).and_return(true)
- allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:relative_archive_path).and_return('test')
- allow(SecureRandom).to receive(:hex).and_return('abcd')
- setup_files
- end
-
- after do
- FileUtils.rm_rf(storage_path)
- end
-
- context 'normal run' do
- before do
- described_class.import(project: build(:project), archive_file: '', shared: shared)
- end
-
- it 'removes symlinks in root folder' do
- expect(File.exist?(symlink_file)).to be false
- end
-
- it 'removes hidden symlinks in root folder' do
- expect(File.exist?(hidden_symlink_file)).to be false
- end
-
- it 'removes evil symlinks in root folder' do
- expect(File.exist?(evil_symlink_file)).to be false
- end
-
- it 'removes symlinks in subfolders' do
- expect(File.exist?(subfolder_symlink_file)).to be false
- end
-
- it 'does not remove a valid file' do
- expect(File.exist?(valid_file)).to be true
- end
-
- it 'creates the file in the right subfolder' do
- expect(shared.export_path).to include('test/abcd')
- end
- end
-
- context 'error' do
- before do
- allow_any_instance_of(described_class).to receive(:wait_for_archived_file).and_raise(StandardError)
- described_class.import(project: build(:project), archive_file: '', shared: shared)
- end
-
- it 'removes symlinks in root folder' do
- expect(File.exist?(symlink_file)).to be false
- end
-
- it 'removes hidden symlinks in root folder' do
- expect(File.exist?(hidden_symlink_file)).to be false
- end
-
- it 'removes symlinks in subfolders' do
- expect(File.exist?(subfolder_symlink_file)).to be false
- end
-
- it 'does not remove a valid file' do
- expect(File.exist?(valid_file)).to be true
- end
- end
-
- def setup_files
- FileUtils.mkdir_p("#{shared.export_path}/subfolder/")
- FileUtils.touch(valid_file)
- FileUtils.ln_s(valid_file, symlink_file)
- FileUtils.ln_s(valid_file, subfolder_symlink_file)
- FileUtils.ln_s(valid_file, hidden_symlink_file)
- FileUtils.ln_s(valid_file, evil_symlink_file)
- end
-end
diff --git a/spec/lib/gitlab/import_export/file_importer_spec.rb b/spec/lib/gitlab/import_export/file_importer_spec.rb
index 78fccdf1dfc..bf34cefe18f 100644
--- a/spec/lib/gitlab/import_export/file_importer_spec.rb
+++ b/spec/lib/gitlab/import_export/file_importer_spec.rb
@@ -2,7 +2,7 @@ require 'spec_helper'
describe Gitlab::ImportExport::FileImporter do
let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
- let(:export_path) { "#{Dir.tmpdir}/file_importer_spec" }
+ let(:storage_path) { "#{Dir.tmpdir}/file_importer_spec" }
let(:valid_file) { "#{shared.export_path}/valid.json" }
let(:symlink_file) { "#{shared.export_path}/invalid.json" }
let(:hidden_symlink_file) { "#{shared.export_path}/.hidden" }
@@ -11,7 +11,9 @@ describe Gitlab::ImportExport::FileImporter do
before do
stub_const('Gitlab::ImportExport::FileImporter::MAX_RETRIES', 0)
- allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+ stub_uploads_object_storage(FileUploader)
+
+ allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(storage_path)
allow_any_instance_of(Gitlab::ImportExport::CommandLineUtil).to receive(:untar_zxf).and_return(true)
allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:relative_archive_path).and_return('test')
allow(SecureRandom).to receive(:hex).and_return('abcd')
@@ -19,12 +21,12 @@ describe Gitlab::ImportExport::FileImporter do
end
after do
- FileUtils.rm_rf(export_path)
+ FileUtils.rm_rf(storage_path)
end
context 'normal run' do
before do
- described_class.import(project: nil, archive_file: '', shared: shared)
+ described_class.import(project: build(:project), archive_file: '', shared: shared)
end
it 'removes symlinks in root folder' do
@@ -55,7 +57,7 @@ describe Gitlab::ImportExport::FileImporter do
context 'error' do
before do
allow_any_instance_of(described_class).to receive(:wait_for_archived_file).and_raise(StandardError)
- described_class.import(project: nil, archive_file: '', shared: shared)
+ described_class.import(project: build(:project), archive_file: '', shared: shared)
end
it 'removes symlinks in root folder' do
diff --git a/spec/lib/gitlab/import_export/importer_object_storage_spec.rb b/spec/lib/gitlab/import_export/importer_object_storage_spec.rb
deleted file mode 100644
index 24a994b3611..00000000000
--- a/spec/lib/gitlab/import_export/importer_object_storage_spec.rb
+++ /dev/null
@@ -1,115 +0,0 @@
-require 'spec_helper'
-
-describe Gitlab::ImportExport::Importer do
- let(:user) { create(:user) }
- let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
- let(:shared) { project.import_export_shared }
- let(:project) { create(:project) }
- let(:import_file) { fixture_file_upload('spec/features/projects/import_export/test_project_export.tar.gz') }
-
- subject(:importer) { described_class.new(project) }
-
- before do
- allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
- allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file)
- stub_feature_flags(import_export_object_storage: true)
- stub_uploads_object_storage(FileUploader)
-
- FileUtils.mkdir_p(shared.export_path)
- ImportExportUpload.create(project: project, import_file: import_file)
- end
-
- after do
- FileUtils.rm_rf(test_path)
- end
-
- describe '#execute' do
- it 'succeeds' do
- importer.execute
-
- expect(shared.errors).to be_empty
- end
-
- it 'extracts the archive' do
- expect(Gitlab::ImportExport::FileImporter).to receive(:import).and_call_original
-
- importer.execute
- end
-
- it 'checks the version' do
- expect(Gitlab::ImportExport::VersionChecker).to receive(:check!).and_call_original
-
- importer.execute
- end
-
- context 'all restores are executed' do
- [
- Gitlab::ImportExport::AvatarRestorer,
- Gitlab::ImportExport::RepoRestorer,
- Gitlab::ImportExport::WikiRestorer,
- Gitlab::ImportExport::UploadsRestorer,
- Gitlab::ImportExport::LfsRestorer,
- Gitlab::ImportExport::StatisticsRestorer
- ].each do |restorer|
- it "calls the #{restorer}" do
- fake_restorer = double(restorer.to_s)
-
- expect(fake_restorer).to receive(:restore).and_return(true).at_least(1)
- expect(restorer).to receive(:new).and_return(fake_restorer).at_least(1)
-
- importer.execute
- end
- end
-
- it 'restores the ProjectTree' do
- expect(Gitlab::ImportExport::ProjectTreeRestorer).to receive(:new).and_call_original
-
- importer.execute
- end
-
- it 'removes the import file' do
- expect(importer).to receive(:remove_import_file).and_call_original
-
- importer.execute
-
- expect(project.import_export_upload.import_file&.file).to be_nil
- end
- end
-
- context 'when project successfully restored' do
- let!(:existing_project) { create(:project, namespace: user.namespace) }
- let(:project) { create(:project, namespace: user.namespace, name: 'whatever', path: 'whatever') }
-
- before do
- restorers = double(:restorers, all?: true)
-
- allow(subject).to receive(:import_file).and_return(true)
- allow(subject).to receive(:check_version!).and_return(true)
- allow(subject).to receive(:restorers).and_return(restorers)
- allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path }))
- end
-
- context 'when import_data' do
- context 'has original_path' do
- it 'overwrites existing project' do
- expect_any_instance_of(::Projects::OverwriteProjectService).to receive(:execute).with(existing_project)
-
- subject.execute
- end
- end
-
- context 'has not original_path' do
- before do
- allow(project).to receive(:import_data).and_return(double(data: {}))
- end
-
- it 'does not call the overwrite service' do
- expect_any_instance_of(::Projects::OverwriteProjectService).not_to receive(:execute).with(existing_project)
-
- subject.execute
- end
- end
- end
- end
- end
-end
diff --git a/spec/lib/gitlab/import_export/importer_spec.rb b/spec/lib/gitlab/import_export/importer_spec.rb
index 8053c48ad6c..11f98d782b1 100644
--- a/spec/lib/gitlab/import_export/importer_spec.rb
+++ b/spec/lib/gitlab/import_export/importer_spec.rb
@@ -4,16 +4,18 @@ describe Gitlab::ImportExport::Importer do
let(:user) { create(:user) }
let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
let(:shared) { project.import_export_shared }
- let(:project) { create(:project, import_source: File.join(test_path, 'test_project_export.tar.gz')) }
+ let(:project) { create(:project) }
+ let(:import_file) { fixture_file_upload('spec/features/projects/import_export/test_project_export.tar.gz') }
subject(:importer) { described_class.new(project) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file)
+ stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(shared.export_path)
- FileUtils.cp(Rails.root.join('spec/features/projects/import_export/test_project_export.tar.gz'), test_path)
+ ImportExportUpload.create(project: project, import_file: import_file)
end
after do
@@ -64,6 +66,14 @@ describe Gitlab::ImportExport::Importer do
importer.execute
end
+ it 'removes the import file' do
+ expect(importer).to receive(:remove_import_file).and_call_original
+
+ importer.execute
+
+ expect(project.import_export_upload.import_file&.file).to be_nil
+ end
+
it 'sets the correct visibility_level when visibility level is a string' do
project.create_or_update_import_data(
data: { override_params: { visibility_level: Gitlab::VisibilityLevel::PRIVATE.to_s } }
@@ -85,7 +95,6 @@ describe Gitlab::ImportExport::Importer do
allow(subject).to receive(:import_file).and_return(true)
allow(subject).to receive(:check_version!).and_return(true)
allow(subject).to receive(:restorers).and_return(restorers)
- allow(restorers).to receive(:all?).and_return(true)
allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path }))
end
diff --git a/spec/lib/gitlab/import_export/saver_spec.rb b/spec/lib/gitlab/import_export/saver_spec.rb
index 02f1a4b81aa..d185ff2dfcc 100644
--- a/spec/lib/gitlab/import_export/saver_spec.rb
+++ b/spec/lib/gitlab/import_export/saver_spec.rb
@@ -18,26 +18,12 @@ describe Gitlab::ImportExport::Saver do
FileUtils.rm_rf(export_path)
end
- context 'local archive' do
- it 'saves the repo to disk' do
- stub_feature_flags(import_export_object_storage: false)
+ it 'saves the repo using object storage' do
+ stub_uploads_object_storage(ImportExportUploader)
- subject.save
+ subject.save
- expect(shared.errors).to be_empty
- expect(Dir.empty?(shared.archive_path)).to be false
- end
- end
-
- context 'object storage' do
- it 'saves the repo using object storage' do
- stub_feature_flags(import_export_object_storage: true)
- stub_uploads_object_storage(ImportExportUploader)
-
- subject.save
-
- expect(ImportExportUpload.find_by(project: project).export_file.url)
- .to match(%r[\/uploads\/-\/system\/import_export_upload\/export_file.*])
- end
+ expect(ImportExportUpload.find_by(project: project).export_file.url)
+ .to match(%r[\/uploads\/-\/system\/import_export_upload\/export_file.*])
end
end
diff --git a/spec/lib/gitlab/import_export/uploads_manager_spec.rb b/spec/lib/gitlab/import_export/uploads_manager_spec.rb
index f799de18cd0..792117e1df1 100644
--- a/spec/lib/gitlab/import_export/uploads_manager_spec.rb
+++ b/spec/lib/gitlab/import_export/uploads_manager_spec.rb
@@ -4,6 +4,7 @@ describe Gitlab::ImportExport::UploadsManager do
let(:shared) { project.import_export_shared }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:project) { create(:project) }
+ let(:upload) { create(:upload, :issuable_upload, :object_storage, model: project) }
let(:exported_file_path) { "#{shared.export_path}/uploads/#{upload.secret}/#{File.basename(upload.path)}" }
subject(:manager) { described_class.new(project: project, shared: shared) }
@@ -69,44 +70,20 @@ describe Gitlab::ImportExport::UploadsManager do
end
end
end
+ end
- context 'using object storage' do
- let!(:upload) { create(:upload, :issuable_upload, :object_storage, model: project) }
-
- before do
- stub_feature_flags(import_export_object_storage: true)
- stub_uploads_object_storage(FileUploader)
- end
-
- it 'saves the file' do
- fake_uri = double
-
- expect(fake_uri).to receive(:open).and_return(StringIO.new('File content'))
- expect(URI).to receive(:parse).and_return(fake_uri)
-
- manager.save
+ describe '#restore' do
+ before do
+ stub_uploads_object_storage(FileUploader)
- expect(File.read(exported_file_path)).to eq('File content')
- end
+ FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038'))
+ FileUtils.touch(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038', "dummy.txt"))
end
- describe '#restore' do
- context 'using object storage' do
- before do
- stub_feature_flags(import_export_object_storage: true)
- stub_uploads_object_storage(FileUploader)
-
- FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038'))
- FileUtils.touch(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038', "dummy.txt"))
- end
+ it 'restores the file' do
+ manager.restore
- it 'restores the file' do
- manager.restore
-
- expect(project.uploads.size).to eq(1)
- expect(project.uploads.first.build_uploader.filename).to eq('dummy.txt')
- end
- end
+ expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt')
end
end
end
diff --git a/spec/lib/gitlab/import_export/uploads_restorer_spec.rb b/spec/lib/gitlab/import_export/uploads_restorer_spec.rb
index acef97459b8..6072f18b8c7 100644
--- a/spec/lib/gitlab/import_export/uploads_restorer_spec.rb
+++ b/spec/lib/gitlab/import_export/uploads_restorer_spec.rb
@@ -8,7 +8,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/random'))
- FileUtils.touch(File.join(shared.export_path, 'uploads/random', "dummy.txt"))
+ FileUtils.touch(File.join(shared.export_path, 'uploads/random', 'dummy.txt'))
end
after do
@@ -27,9 +27,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
it 'copies the uploads to the project path' do
subject.restore
- uploads = Dir.glob(File.join(subject.uploads_path, '**/*')).map { |file| File.basename(file) }
-
- expect(uploads).to include('dummy.txt')
+ expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt')
end
end
@@ -45,9 +43,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
it 'copies the uploads to the project path' do
subject.restore
- uploads = Dir.glob(File.join(subject.uploads_path, '**/*')).map { |file| File.basename(file) }
-
- expect(uploads).to include('dummy.txt')
+ expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt')
end
end
end
diff --git a/spec/lib/gitlab/import_export/uploads_saver_spec.rb b/spec/lib/gitlab/import_export/uploads_saver_spec.rb
index c716edd9397..24993460e51 100644
--- a/spec/lib/gitlab/import_export/uploads_saver_spec.rb
+++ b/spec/lib/gitlab/import_export/uploads_saver_spec.rb
@@ -7,7 +7,6 @@ describe Gitlab::ImportExport::UploadsSaver do
let(:shared) { project.import_export_shared }
before do
- stub_feature_flags(import_export_object_storage: false)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
end
diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb
index 9b7f932ec3a..3649990670b 100644
--- a/spec/models/namespace_spec.rb
+++ b/spec/models/namespace_spec.rb
@@ -394,12 +394,6 @@ describe Namespace do
child.destroy
end
end
-
- it 'removes the exports folder' do
- expect(namespace).to receive(:remove_exports!)
-
- namespace.destroy
- end
end
context 'hashed storage' do
@@ -414,12 +408,6 @@ describe Namespace do
expect(File.exist?(deleted_path_in_dir)).to be(false)
end
-
- it 'removes the exports folder' do
- expect(namespace).to receive(:remove_exports!)
-
- namespace.destroy
- end
end
end
@@ -706,26 +694,6 @@ describe Namespace do
end
end
- describe '#remove_exports' do
- let(:legacy_project) { create(:project, :with_export, :legacy_storage, namespace: namespace) }
- let(:hashed_project) { create(:project, :with_export, namespace: namespace) }
- let(:export_path) { Dir.mktmpdir('namespace_remove_exports_spec') }
- let(:legacy_export) { legacy_project.export_project_path }
- let(:hashed_export) { hashed_project.export_project_path }
-
- it 'removes exports for legacy and hashed projects' do
- allow(Gitlab::ImportExport).to receive(:storage_path) { export_path }
-
- expect(File.exist?(legacy_export)).to be_truthy
- expect(File.exist?(hashed_export)).to be_truthy
-
- namespace.remove_exports!
-
- expect(File.exist?(legacy_export)).to be_falsy
- expect(File.exist?(hashed_export)).to be_falsy
- end
- end
-
describe '#full_path_was' do
context 'when the group has no parent' do
it 'should return the path was' do
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index 264632dba4b..809a21e80ee 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -2854,73 +2854,12 @@ describe Project do
end
describe '#remove_export' do
- let(:legacy_project) { create(:project, :legacy_storage, :with_export) }
let(:project) { create(:project, :with_export) }
- before do
- stub_feature_flags(import_export_object_storage: false)
- end
-
- it 'removes the exports directory for the project' do
- expect(File.exist?(project.export_path)).to be_truthy
-
- allow(FileUtils).to receive(:rm_rf).and_call_original
- expect(FileUtils).to receive(:rm_rf).with(project.export_path).and_call_original
+ it 'removes the export' do
project.remove_exports
- expect(File.exist?(project.export_path)).to be_falsy
- end
-
- it 'is a no-op on legacy projects when there is no namespace' do
- export_path = legacy_project.export_path
-
- legacy_project.namespace.delete
- legacy_project.reload
-
- expect(FileUtils).not_to receive(:rm_rf).with(export_path)
-
- legacy_project.remove_exports
-
- expect(File.exist?(export_path)).to be_truthy
- end
-
- it 'runs on hashed storage projects when there is no namespace' do
- export_path = project.export_path
-
- project.namespace.delete
- legacy_project.reload
-
- allow(FileUtils).to receive(:rm_rf).and_call_original
- expect(FileUtils).to receive(:rm_rf).with(export_path).and_call_original
-
- project.remove_exports
-
- expect(File.exist?(export_path)).to be_falsy
- end
-
- it 'is run when the project is destroyed' do
- expect(project).to receive(:remove_exports).and_call_original
-
- project.destroy
- end
- end
-
- describe '#remove_exported_project_file' do
- let(:project) { create(:project, :with_export) }
-
- it 'removes the exported project file' do
- stub_feature_flags(import_export_object_storage: false)
-
- exported_file = project.export_project_path
-
- expect(File.exist?(exported_file)).to be_truthy
-
- allow(FileUtils).to receive(:rm_rf).and_call_original
- expect(FileUtils).to receive(:rm_rf).with(exported_file).and_call_original
-
- project.remove_exported_project_file
-
- expect(File.exist?(exported_file)).to be_falsy
+ expect(project.export_project_object_exists?).to be_falsey
end
end
diff --git a/spec/requests/api/project_export_spec.rb b/spec/requests/api/project_export_spec.rb
index 45e4e35d773..0d4854a83b9 100644
--- a/spec/requests/api/project_export_spec.rb
+++ b/spec/requests/api/project_export_spec.rb
@@ -4,8 +4,8 @@ describe API::ProjectExport do
set(:project) { create(:project) }
set(:project_none) { create(:project) }
set(:project_started) { create(:project) }
- set(:project_finished) { create(:project) }
- set(:project_after_export) { create(:project) }
+ let(:project_finished) { create(:project, :with_export) }
+ let(:project_after_export) { create(:project, :with_export) }
set(:user) { create(:user) }
set(:admin) { create(:admin) }
@@ -29,13 +29,7 @@ describe API::ProjectExport do
# simulate exporting work directory
FileUtils.mkdir_p File.join(project_started.export_path, 'securerandom-hex')
- # simulate exported
- FileUtils.mkdir_p project_finished.export_path
- FileUtils.touch File.join(project_finished.export_path, '_export.tar.gz')
-
# simulate in after export action
- FileUtils.mkdir_p project_after_export.export_path
- FileUtils.touch File.join(project_after_export.export_path, '_export.tar.gz')
FileUtils.touch Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project_after_export)
end
@@ -191,14 +185,11 @@ describe API::ProjectExport do
context 'when upload complete' do
before do
- FileUtils.rm_rf(project_after_export.export_path)
-
- if project_after_export.export_project_object_exists?
- upload = project_after_export.import_export_upload
+ project_after_export.remove_exports
+ end
- upload.remove_export_file!
- upload.save
- end
+ it 'has removed the export' do
+ expect(project_after_export.export_project_object_exists?).to be_falsey
end
it_behaves_like '404 response' do
@@ -273,13 +264,13 @@ describe API::ProjectExport do
before do
stub_uploads_object_storage(ImportExportUploader)
- [project, project_finished, project_after_export].each do |p|
- p.add_maintainer(user)
+ project.add_maintainer(user)
+ project_finished.add_maintainer(user)
+ project_after_export.add_maintainer(user)
- upload = ImportExportUpload.new(project: p)
- upload.export_file = fixture_file_upload('spec/fixtures/project_export.tar.gz', "`/tar.gz")
- upload.save!
- end
+ upload = ImportExportUpload.new(project: project)
+ upload.export_file = fixture_file_upload('spec/fixtures/project_export.tar.gz', "`/tar.gz")
+ upload.save!
end
it_behaves_like 'get project download by strategy'
diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb
index bc06f3c3732..c8fa4754810 100644
--- a/spec/requests/api/project_import_spec.rb
+++ b/spec/requests/api/project_import_spec.rb
@@ -7,7 +7,6 @@ describe API::ProjectImport do
let(:namespace) { create(:group) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
- stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
namespace.add_owner(user)
diff --git a/spec/support/import_export/export_file_helper.rb b/spec/support/import_export/export_file_helper.rb
index 4d925ac77f4..cbe238e20ea 100644
--- a/spec/support/import_export/export_file_helper.rb
+++ b/spec/support/import_export/export_file_helper.rb
@@ -52,7 +52,7 @@ module ExportFileHelper
# Expands the compressed file for an exported project into +tmpdir+
def in_directory_with_expanded_export(project)
Dir.mktmpdir do |tmpdir|
- export_file = project.export_project_path
+ export_file = project.import_export_upload.export_file.path
_output, exit_status = Gitlab::Popen.popen(%W{tar -zxf #{export_file} -C #{tmpdir}})
yield(exit_status, tmpdir)