diff options
Diffstat (limited to 'lib/gitlab/import_export')
-rw-r--r-- | lib/gitlab/import_export/decompressed_archive_size_validator.rb | 83 | ||||
-rw-r--r-- | lib/gitlab/import_export/design_repo_restorer.rb | 7 | ||||
-rw-r--r-- | lib/gitlab/import_export/design_repo_saver.rb | 12 | ||||
-rw-r--r-- | lib/gitlab/import_export/file_importer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/group/tree_restorer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/importer.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/import_export/repo_restorer.rb | 21 | ||||
-rw-r--r-- | lib/gitlab/import_export/repo_saver.rb | 20 | ||||
-rw-r--r-- | lib/gitlab/import_export/saver.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/import_export/wiki_repo_saver.rb | 15 |
10 files changed, 99 insertions, 75 deletions
diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 219821a7150..37f1bdc3009 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -1,24 +1,16 @@ # frozen_string_literal: true -require 'zlib' - module Gitlab module ImportExport class DecompressedArchiveSizeValidator include Gitlab::Utils::StrongMemoize DEFAULT_MAX_BYTES = 10.gigabytes.freeze - CHUNK_SIZE = 4096.freeze - - attr_reader :error + TIMEOUT_LIMIT = 60.seconds def initialize(archive_path:, max_bytes: self.class.max_bytes) @archive_path = archive_path @max_bytes = max_bytes - @bytes_read = 0 - @total_reads = 0 - @denominator = 5 - @error = nil end def valid? @@ -31,59 +23,62 @@ module Gitlab DEFAULT_MAX_BYTES end - def archive_file - @archive_file ||= File.open(@archive_path) - end - private def validate - until archive_file.eof? - compressed_chunk = archive_file.read(CHUNK_SIZE) + pgrp = nil + valid_archive = true - inflate_stream.inflate(compressed_chunk) do |chunk| - @bytes_read += chunk.size - @total_reads += 1 - end + Timeout.timeout(TIMEOUT_LIMIT) do + stdin, stdout, stderr, wait_thr = Open3.popen3(command, pgroup: true) + stdin.close + pgrp = Process.getpgid(wait_thr[:pid]) + status = wait_thr.value - # Start garbage collection every 5 reads in order - # to prevent memory bloat during archive decompression - GC.start if gc_start? + if status.success? + result = stdout.readline - if @bytes_read > @max_bytes - @error = error_message + if result.to_i > @max_bytes + valid_archive = false - return false + log_error('Decompressed archive size limit reached') + end + else + valid_archive = false + + log_error(stderr.readline) end + + ensure + stdout.close + stderr.close end - true - rescue => e - @error = error_message + valid_archive + rescue Timeout::Error + log_error('Timeout reached during archive decompression') - Gitlab::ErrorTracking.track_exception(e) - - Gitlab::Import::Logger.info( - message: @error, - error: e.message - ) + Process.kill(-1, pgrp) if pgrp false - ensure - inflate_stream.close - archive_file.close - end + rescue => e + log_error(e.message) - def inflate_stream - @inflate_stream ||= Zlib::Inflate.new(Zlib::MAX_WBITS + 32) + Process.kill(-1, pgrp) if pgrp + + false end - def gc_start? - @total_reads % @denominator == 0 + def command + "gzip -dc #{@archive_path} | wc -c" end - def error_message - _('Decompressed archive size validation failed.') + def log_error(error) + Gitlab::Import::Logger.info( + message: error, + import_upload_archive_path: @archive_path, + import_upload_archive_size: File.size(@archive_path) + ) end end end diff --git a/lib/gitlab/import_export/design_repo_restorer.rb b/lib/gitlab/import_export/design_repo_restorer.rb index a702c58a7c2..e093b4b0697 100644 --- a/lib/gitlab/import_export/design_repo_restorer.rb +++ b/lib/gitlab/import_export/design_repo_restorer.rb @@ -3,10 +3,11 @@ module Gitlab module ImportExport class DesignRepoRestorer < RepoRestorer - def initialize(project:, shared:, path_to_bundle:) - super(project: project, shared: shared, path_to_bundle: path_to_bundle) + extend ::Gitlab::Utils::Override - @repository = project.design_repository + override :repository + def repository + @repository ||= importable.design_repository end # `restore` method is handled in super class diff --git a/lib/gitlab/import_export/design_repo_saver.rb b/lib/gitlab/import_export/design_repo_saver.rb index db9ebee6a13..b400aedc205 100644 --- a/lib/gitlab/import_export/design_repo_saver.rb +++ b/lib/gitlab/import_export/design_repo_saver.rb @@ -3,16 +3,18 @@ module Gitlab module ImportExport class DesignRepoSaver < RepoSaver - def save - @repository = project.design_repository + extend ::Gitlab::Utils::Override - super + override :repository + def repository + @repository ||= exportable.design_repository end private - def bundle_full_path - File.join(shared.export_path, ::Gitlab::ImportExport.design_repo_bundle_filename) + override :bundle_filename + def bundle_filename + ::Gitlab::ImportExport.design_repo_bundle_filename end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 5a6f6e017d2..51d58aae54f 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -87,7 +87,7 @@ module Gitlab end def validate_decompressed_archive_size - raise ImporterError.new(size_validator.error) unless size_validator.valid? + raise ImporterError.new(_('Decompressed archive size validation failed.')) unless size_validator.valid? end def size_validator diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index dfe27118d66..925ab6680ba 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -6,7 +6,7 @@ module Gitlab class TreeRestorer include Gitlab::Utils::StrongMemoize - attr_reader :user, :shared + attr_reader :user, :shared, :groups_mapping def initialize(user:, shared:, group:) @user = user diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 789249c7d91..390909efe36 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -75,19 +75,19 @@ module Gitlab def repo_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, shared: shared, - project: project) + importable: project) end def wiki_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, shared: shared, - project: ProjectWiki.new(project)) + importable: ProjectWiki.new(project)) end def design_repo_restorer Gitlab::ImportExport::DesignRepoRestorer.new(path_to_bundle: design_repo_path, shared: shared, - project: project) + importable: project) end def uploads_restorer diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index f808e30bd6e..8af7b68d78e 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -5,10 +5,12 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:, path_to_bundle:) - @repository = project.repository + attr_reader :importable + + def initialize(importable:, shared:, path_to_bundle:) @path_to_bundle = path_to_bundle @shared = shared + @importable = importable end def restore @@ -17,14 +19,25 @@ module Gitlab ensure_repository_does_not_exist! repository.create_from_bundle(path_to_bundle) + update_importable_repository_info + + true rescue => e shared.error(e) false end + def repository + @repository ||= importable.repository + end + private - attr_accessor :repository, :path_to_bundle, :shared + attr_accessor :path_to_bundle, :shared + + def update_importable_repository_info + # No-op. Overridden in EE + end def ensure_repository_does_not_exist! if repository.exists? @@ -38,3 +51,5 @@ module Gitlab end end end + +Gitlab::ImportExport::RepoRestorer.prepend_if_ee('EE::Gitlab::ImportExport::RepoRestorer') diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 898cd7898ba..0fdd0722b65 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -5,12 +5,11 @@ module Gitlab class RepoSaver include Gitlab::ImportExport::CommandLineUtil - attr_reader :project, :repository, :shared + attr_reader :exportable, :shared - def initialize(project:, shared:) - @project = project + def initialize(exportable:, shared:) + @exportable = exportable @shared = shared - @repository = @project.repository end def save @@ -19,6 +18,10 @@ module Gitlab bundle_to_disk end + def repository + @repository ||= @exportable.repository + end + private def repository_exists? @@ -26,11 +29,16 @@ module Gitlab end def bundle_full_path - File.join(shared.export_path, ImportExport.project_bundle_filename) + File.join(shared.export_path, bundle_filename) + end + + def bundle_filename + ::Gitlab::ImportExport.project_bundle_filename end def bundle_to_disk - mkdir_p(shared.export_path) + mkdir_p(File.dirname(bundle_full_path)) + repository.bundle_to_disk(bundle_full_path) rescue => e shared.error(e) diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 045ba2495bf..bb2bbda4bd6 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -31,7 +31,7 @@ module Gitlab @shared.error(e) false ensure - remove_base_tmp_dir + remove_archive_tmp_dir end private @@ -40,8 +40,8 @@ module Gitlab tar_czf(archive: archive_file, dir: @shared.export_path) end - def remove_base_tmp_dir - FileUtils.rm_rf(@shared.base_path) + def remove_archive_tmp_dir + FileUtils.rm_rf(@shared.archive_path) end def archive_file diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 93ae6f6b02a..4b1cf4915e4 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -3,18 +3,21 @@ module Gitlab module ImportExport class WikiRepoSaver < RepoSaver - def save - wiki = ProjectWiki.new(project) - @repository = wiki.repository + extend ::Gitlab::Utils::Override - super + override :repository + def repository + @repository ||= exportable.wiki.repository end private - def bundle_full_path - File.join(shared.export_path, ImportExport.wiki_repo_bundle_filename) + override :bundle_filename + def bundle_filename + ::Gitlab::ImportExport.wiki_repo_bundle_filename end end end end + +Gitlab::ImportExport::WikiRepoSaver.prepend_if_ee('EE::Gitlab::ImportExport::WikiRepoSaver') |