From 55ad5423eea6281657e1f7c6630774934f1f26b9 Mon Sep 17 00:00:00 2001 From: Luke Duncalfe Date: Tue, 16 Jul 2019 15:31:44 +1200 Subject: CE-specific changes to allow design repo exports https://gitlab.com/gitlab-org/gitlab-ee/issues/11090 --- app/models/repository.rb | 2 + .../projects/import_export/export_service.rb | 36 +++++++++------- lib/gitlab/import_export.rb | 4 ++ lib/gitlab/import_export/importer.rb | 48 +++++++++++----------- lib/gitlab/import_export/repo_restorer.rb | 12 ++++-- lib/gitlab/import_export/repo_saver.rb | 20 ++++++--- lib/gitlab/import_export/wiki_repo_saver.rb | 22 +++------- lib/gitlab/import_export/wiki_restorer.rb | 7 +++- spec/lib/gitlab/import_export/repo_saver_spec.rb | 12 +++++- .../gitlab/import_export/wiki_repo_saver_spec.rb | 12 +++++- spec/models/repository_spec.rb | 16 +++++++- .../projects/import_export/export_service_spec.rb | 21 ++++++---- 12 files changed, 132 insertions(+), 80 deletions(-) diff --git a/app/models/repository.rb b/app/models/repository.rb index 5cb4b56a114..488ac9b72df 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -1108,6 +1108,8 @@ class Repository raw.create_repository after_create + + true end def blobs_metadata(paths, ref = 'HEAD') diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb index 9c6d7ef41f6..495e240c6ab 100644 --- a/app/services/projects/import_export/export_service.rb +++ b/app/services/projects/import_export/export_service.rb @@ -12,6 +12,8 @@ module Projects private + attr_accessor :shared + def execute_after_export_action(after_export_strategy) return unless after_export_strategy @@ -21,50 +23,54 @@ module Projects end def save_all! - if save_services - Gitlab::ImportExport::Saver.save(project: project, shared: @shared) + if save_exporters + Gitlab::ImportExport::Saver.save(project: project, shared: shared) notify_success else cleanup_and_notify_error! end end - def save_services - [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver].all?(&:save) + def save_exporters + exporters.all?(&:save) + end + + def exporters + [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver] end def version_saver - Gitlab::ImportExport::VersionSaver.new(shared: @shared) + Gitlab::ImportExport::VersionSaver.new(shared: shared) end def avatar_saver - Gitlab::ImportExport::AvatarSaver.new(project: project, shared: @shared) + Gitlab::ImportExport::AvatarSaver.new(project: project, shared: shared) end def project_tree_saver - Gitlab::ImportExport::ProjectTreeSaver.new(project: project, current_user: @current_user, shared: @shared, params: @params) + Gitlab::ImportExport::ProjectTreeSaver.new(project: project, current_user: current_user, shared: shared, params: params) end def uploads_saver - Gitlab::ImportExport::UploadsSaver.new(project: project, shared: @shared) + Gitlab::ImportExport::UploadsSaver.new(project: project, shared: shared) end def repo_saver - Gitlab::ImportExport::RepoSaver.new(project: project, shared: @shared) + Gitlab::ImportExport::RepoSaver.new(project: project, shared: shared) end def wiki_repo_saver - Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared) + Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: shared) end def lfs_saver - Gitlab::ImportExport::LfsSaver.new(project: project, shared: @shared) + Gitlab::ImportExport::LfsSaver.new(project: project, shared: shared) end def cleanup_and_notify_error - Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{@shared.errors.join(', ')}") # rubocop:disable Gitlab/RailsLogger + Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{shared.errors.join(', ')}") # rubocop:disable Gitlab/RailsLogger - FileUtils.rm_rf(@shared.export_path) + FileUtils.rm_rf(shared.export_path) notify_error end @@ -72,7 +78,7 @@ module Projects def cleanup_and_notify_error! cleanup_and_notify_error - raise Gitlab::ImportExport::Error.new(@shared.errors.join(', ')) + raise Gitlab::ImportExport::Error.new(shared.errors.to_sentence) end def notify_success @@ -80,7 +86,7 @@ module Projects end def notify_error - notification_service.project_not_exported(@project, @current_user, @shared.errors) + notification_service.project_not_exported(project, current_user, shared.errors) end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index d08848a65a8..36bcbd4ef48 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -38,6 +38,10 @@ module Gitlab "lfs-objects" end + def wiki_repo_bundle_filename + "project.wiki.bundle" + end + def config_file Rails.root.join('lib/gitlab/import_export/import_export.yml') end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 767f1b5de0e..5ab91e6f2dc 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -21,7 +21,7 @@ module Gitlab if import_file && check_version! && restorers.all?(&:restore) && overwrite_project project_tree.restored_project else - raise Projects::ImportService::Error.new(@shared.errors.join(', ')) + raise Projects::ImportService::Error.new(shared.errors.to_sentence) end rescue => e raise Projects::ImportService::Error.new(e.message) @@ -31,70 +31,72 @@ module Gitlab private + attr_accessor :archive_file, :current_user, :project, :shared + def restorers [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer, lfs_restorer, statistics_restorer] end def import_file - Gitlab::ImportExport::FileImporter.import(project: @project, - archive_file: @archive_file, - shared: @shared) + Gitlab::ImportExport::FileImporter.import(project: project, + archive_file: archive_file, + shared: shared) end def check_version! - Gitlab::ImportExport::VersionChecker.check!(shared: @shared) + Gitlab::ImportExport::VersionChecker.check!(shared: shared) end def project_tree - @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user, - shared: @shared, - project: @project) + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: current_user, + shared: shared, + project: project) end def avatar_restorer - Gitlab::ImportExport::AvatarRestorer.new(project: project_tree.restored_project, shared: @shared) + Gitlab::ImportExport::AvatarRestorer.new(project: project_tree.restored_project, shared: shared) end def repo_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, - shared: @shared, + shared: shared, project: project_tree.restored_project) end def wiki_restorer Gitlab::ImportExport::WikiRestorer.new(path_to_bundle: wiki_repo_path, - shared: @shared, + shared: shared, project: ProjectWiki.new(project_tree.restored_project), - wiki_enabled: @project.wiki_enabled?) + wiki_enabled: project.wiki_enabled?) end def uploads_restorer - Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) + Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: shared) end def lfs_restorer - Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared) + Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: shared) end def statistics_restorer - Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: @shared) + Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: shared) end def path_with_namespace - File.join(@project.namespace.full_path, @project.path) + File.join(project.namespace.full_path, project.path) end def repo_path - File.join(@shared.export_path, 'project.bundle') + File.join(shared.export_path, Gitlab::ImportExport.project_bundle_filename) end def wiki_repo_path - File.join(@shared.export_path, 'project.wiki.bundle') + File.join(shared.export_path, Gitlab::ImportExport.wiki_repo_bundle_filename) end def remove_import_file - upload = @project.import_export_upload + upload = project.import_export_upload return unless upload&.import_file&.file @@ -105,10 +107,10 @@ module Gitlab def overwrite_project project = project_tree.restored_project - return unless can?(@current_user, :admin_namespace, project.namespace) + return unless can?(current_user, :admin_namespace, project.namespace) if overwrite_project? - ::Projects::OverwriteProjectService.new(project, @current_user) + ::Projects::OverwriteProjectService.new(project, current_user) .execute(project_to_overwrite) end @@ -116,7 +118,7 @@ module Gitlab end def original_path - @project.import_data&.data&.fetch('original_path', nil) + project.import_data&.data&.fetch('original_path', nil) end def overwrite_project? @@ -125,7 +127,7 @@ module Gitlab def project_to_overwrite strong_memoize(:project_to_overwrite) do - Project.find_by_full_path("#{@project.namespace.full_path}/#{original_path}") + Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}") end end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 91167a9c4fb..3123687453f 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -6,19 +6,23 @@ module Gitlab include Gitlab::ImportExport::CommandLineUtil def initialize(project:, shared:, path_to_bundle:) - @project = project + @repository = project.repository @path_to_bundle = path_to_bundle @shared = shared end def restore - return true unless File.exist?(@path_to_bundle) + return true unless File.exist?(path_to_bundle) - @project.repository.create_from_bundle(@path_to_bundle) + repository.create_from_bundle(path_to_bundle) rescue => e - @shared.error(e) + shared.error(e) false end + + private + + attr_accessor :repository, :path_to_bundle, :shared end end end diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index a60618dfcec..898cd7898ba 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -5,27 +5,35 @@ module Gitlab class RepoSaver include Gitlab::ImportExport::CommandLineUtil - attr_reader :full_path + attr_reader :project, :repository, :shared def initialize(project:, shared:) @project = project @shared = shared + @repository = @project.repository end def save - return true if @project.empty_repo? # it's ok to have no repo + return true unless repository_exists? # it's ok to have no repo - @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) bundle_to_disk end private + def repository_exists? + repository.exists? && !repository.empty? + end + + def bundle_full_path + File.join(shared.export_path, ImportExport.project_bundle_filename) + end + def bundle_to_disk - mkdir_p(@shared.export_path) - @project.repository.bundle_to_disk(@full_path) + mkdir_p(shared.export_path) + repository.bundle_to_disk(bundle_full_path) rescue => e - @shared.error(e) + shared.error(e) false end end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 7303bcf61a4..93ae6f6b02a 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -4,28 +4,16 @@ module Gitlab module ImportExport class WikiRepoSaver < RepoSaver def save - @wiki = ProjectWiki.new(@project) - return true unless wiki_repository_exists? # it's okay to have no Wiki + wiki = ProjectWiki.new(project) + @repository = wiki.repository - bundle_to_disk(File.join(@shared.export_path, project_filename)) - end - - def bundle_to_disk(full_path) - mkdir_p(@shared.export_path) - @wiki.repository.bundle_to_disk(full_path) - rescue => e - @shared.error(e) - false + super end private - def project_filename - "project.wiki.bundle" - end - - def wiki_repository_exists? - @wiki.repository.exists? && !@wiki.repository.empty? + def bundle_full_path + File.join(shared.export_path, ImportExport.wiki_repo_bundle_filename) end end end diff --git a/lib/gitlab/import_export/wiki_restorer.rb b/lib/gitlab/import_export/wiki_restorer.rb index 28b5e7449cd..359ba8ba769 100644 --- a/lib/gitlab/import_export/wiki_restorer.rb +++ b/lib/gitlab/import_export/wiki_restorer.rb @@ -6,19 +6,22 @@ module Gitlab def initialize(project:, shared:, path_to_bundle:, wiki_enabled:) super(project: project, shared: shared, path_to_bundle: path_to_bundle) + @project = project @wiki_enabled = wiki_enabled end def restore - @project.wiki if create_empty_wiki? + project.wiki if create_empty_wiki? super end private + attr_accessor :project, :wiki_enabled + def create_empty_wiki? - !File.exist?(@path_to_bundle) && @wiki_enabled + !File.exist?(path_to_bundle) && wiki_enabled end end end diff --git a/spec/lib/gitlab/import_export/repo_saver_spec.rb b/spec/lib/gitlab/import_export/repo_saver_spec.rb index 5a646b4aac8..c3df371af43 100644 --- a/spec/lib/gitlab/import_export/repo_saver_spec.rb +++ b/spec/lib/gitlab/import_export/repo_saver_spec.rb @@ -2,8 +2,8 @@ require 'spec_helper' describe Gitlab::ImportExport::RepoSaver do describe 'bundle a project Git repo' do - let(:user) { create(:user) } - let!(:project) { create(:project, :public, name: 'searchable_project') } + set(:user) { create(:user) } + let!(:project) { create(:project, :repository) } let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } let(:shared) { project.import_export_shared } let(:bundler) { described_class.new(project: project, shared: shared) } @@ -20,5 +20,13 @@ describe Gitlab::ImportExport::RepoSaver do it 'bundles the repo successfully' do expect(bundler.save).to be true end + + context 'when the repo is empty' do + let!(:project) { create(:project) } + + it 'bundles the repo successfully' do + expect(bundler.save).to be true + end + end end end diff --git a/spec/lib/gitlab/import_export/wiki_repo_saver_spec.rb b/spec/lib/gitlab/import_export/wiki_repo_saver_spec.rb index 441aa1defe6..249afbd23d1 100644 --- a/spec/lib/gitlab/import_export/wiki_repo_saver_spec.rb +++ b/spec/lib/gitlab/import_export/wiki_repo_saver_spec.rb @@ -2,8 +2,8 @@ require 'spec_helper' describe Gitlab::ImportExport::WikiRepoSaver do describe 'bundle a wiki Git repo' do - let(:user) { create(:user) } - let!(:project) { create(:project, :public, :wiki_repo, name: 'searchable_project') } + set(:user) { create(:user) } + let!(:project) { create(:project, :wiki_repo) } let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } let(:shared) { project.import_export_shared } let(:wiki_bundler) { described_class.new(project: project, shared: shared) } @@ -23,5 +23,13 @@ describe Gitlab::ImportExport::WikiRepoSaver do it 'bundles the repo successfully' do expect(wiki_bundler.save).to be true end + + context 'when the repo is empty' do + let!(:project) { create(:project) } + + it 'bundles the repo successfully' do + expect(wiki_bundler.save).to be true + end + end end end diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index 419e1dc2459..b6de5e175dc 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -1075,7 +1075,7 @@ describe Repository do let(:ref) { 'refs/heads/master' } it 'returns nil' do - is_expected.to eq(nil) + is_expected.to be_nil end end @@ -1972,7 +1972,7 @@ describe Repository do it 'returns nil if repo does not exist' do allow(repository).to receive(:root_ref).and_raise(Gitlab::Git::Repository::NoRepository) - expect(repository.avatar).to eq(nil) + expect(repository.avatar).to be_nil end it 'returns the first avatar file found in the repository' do @@ -2623,6 +2623,10 @@ describe Repository do expect { repository.create_if_not_exists }.to change { repository.exists? }.from(false).to(true) end + it 'returns true' do + expect(repository.create_if_not_exists).to eq(true) + end + it 'calls out to the repository client to create a repo' do expect(repository.raw.gitaly_repository_client).to receive(:create_repository) @@ -2637,6 +2641,10 @@ describe Repository do repository.create_if_not_exists end + + it 'returns nil' do + expect(repository.create_if_not_exists).to be_nil + end end context 'when the repository exists but the cache is not up to date' do @@ -2648,6 +2656,10 @@ describe Repository do expect { repository.create_if_not_exists }.not_to raise_error end + + it 'returns nil' do + expect(repository.create_if_not_exists).to be_nil + end end end diff --git a/spec/services/projects/import_export/export_service_spec.rb b/spec/services/projects/import_export/export_service_spec.rb index 404bb55629a..146d656c909 100644 --- a/spec/services/projects/import_export/export_service_spec.rb +++ b/spec/services/projects/import_export/export_service_spec.rb @@ -35,20 +35,27 @@ describe Projects::ImportExport::ExportService do end it 'saves the repo' do + # This spec errors when run against the EE codebase as there will be a third repository + # saved (the EE-specific design repository). + # + # Instead, skip this test when run within EE. There is a spec for the EE-specific design repo + # in the corresponding EE spec. + skip if Gitlab.ee? + # once for the normal repo, once for the wiki expect(Gitlab::ImportExport::RepoSaver).to receive(:new).twice.and_call_original service.execute end - it 'saves the lfs objects' do - expect(Gitlab::ImportExport::LfsSaver).to receive(:new).and_call_original + it 'saves the wiki repo' do + expect(Gitlab::ImportExport::WikiRepoSaver).to receive(:new).and_call_original service.execute end - it 'saves the wiki repo' do - expect(Gitlab::ImportExport::WikiRepoSaver).to receive(:new).and_call_original + it 'saves the lfs objects' do + expect(Gitlab::ImportExport::LfsSaver).to receive(:new).and_call_original service.execute end @@ -98,9 +105,9 @@ describe Projects::ImportExport::ExportService do end end - context 'when saver services fail' do + context 'when saving services fail' do before do - allow(service).to receive(:save_services).and_return(false) + allow(service).to receive(:save_exporters).and_return(false) end after do @@ -122,7 +129,7 @@ describe Projects::ImportExport::ExportService do expect(Rails.logger).to receive(:error) end - it 'the after export strategy is not called' do + it 'does not call the export strategy' do expect(service).not_to receive(:execute_after_export_action) end end -- cgit v1.2.1