diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-08-10 13:31:48 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-08-10 13:31:48 +0000 |
commit | 0a3f1f55493660acfabd198d2e1649a881d8852b (patch) | |
tree | 3edcd287de2cd0f7e72917865a5e5c4111b74ecb /lib | |
parent | ea4766228b5536c83f1917d6058be913472ffa2d (diff) | |
download | gitlab-ce-0a3f1f55493660acfabd198d2e1649a881d8852b.tar.gz |
Add latest changes from gitlab-org/gitlab@13-2-stable-ee
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/import_export/decompressed_archive_size_validator.rb | 90 | ||||
-rw-r--r-- | lib/gitlab/import_export/file_importer.rb | 9 |
2 files changed, 99 insertions, 0 deletions
diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb new file mode 100644 index 00000000000..219821a7150 --- /dev/null +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require 'zlib' + +module Gitlab + module ImportExport + class DecompressedArchiveSizeValidator + include Gitlab::Utils::StrongMemoize + + DEFAULT_MAX_BYTES = 10.gigabytes.freeze + CHUNK_SIZE = 4096.freeze + + attr_reader :error + + def initialize(archive_path:, max_bytes: self.class.max_bytes) + @archive_path = archive_path + @max_bytes = max_bytes + @bytes_read = 0 + @total_reads = 0 + @denominator = 5 + @error = nil + end + + def valid? + strong_memoize(:valid) do + validate + end + end + + def self.max_bytes + DEFAULT_MAX_BYTES + end + + def archive_file + @archive_file ||= File.open(@archive_path) + end + + private + + def validate + until archive_file.eof? + compressed_chunk = archive_file.read(CHUNK_SIZE) + + inflate_stream.inflate(compressed_chunk) do |chunk| + @bytes_read += chunk.size + @total_reads += 1 + end + + # Start garbage collection every 5 reads in order + # to prevent memory bloat during archive decompression + GC.start if gc_start? + + if @bytes_read > @max_bytes + @error = error_message + + return false + end + end + + true + rescue => e + @error = error_message + + Gitlab::ErrorTracking.track_exception(e) + + Gitlab::Import::Logger.info( + message: @error, + error: e.message + ) + + false + ensure + inflate_stream.close + archive_file.close + end + + def inflate_stream + @inflate_stream ||= Zlib::Inflate.new(Zlib::MAX_WBITS + 32) + end + + def gc_start? + @total_reads % @denominator == 0 + end + + def error_message + _('Decompressed archive size validation failed.') + end + end + end +end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 9d04d55770d..3cb1eb72ceb 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -28,6 +28,7 @@ module Gitlab copy_archive wait_for_archived_file do + validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: true) decompress_archive end rescue => e @@ -82,6 +83,14 @@ module Gitlab def extracted_files Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) } end + + def validate_decompressed_archive_size + raise ImporterError.new(size_validator.error) unless size_validator.valid? + end + + def size_validator + @size_validator ||= DecompressedArchiveSizeValidator.new(archive_path: @archive_file) + end end end end |