From b6254abd9bce19a80a4720e82761cbafb627c3ef Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 5 Aug 2020 19:14:39 +0000 Subject: Add latest changes from gitlab-org/gitlab@12-10-stable-ee --- .../security-fix-import-decompr-issue.yml | 5 ++ ...ject_import_decompressed_archive_size_limits.md | 28 +++++++ .../decompressed_archive_size_validator.rb | 90 ++++++++++++++++++++++ lib/gitlab/import_export/file_importer.rb | 9 +++ locale/gitlab.pot | 3 + .../decompressed_archive_size_validator_spec.rb | 58 ++++++++++++++ .../lib/gitlab/import_export/file_importer_spec.rb | 39 ++++++++++ vendor/gitignore/C++.gitignore | 0 vendor/gitignore/Java.gitignore | 0 9 files changed, 232 insertions(+) create mode 100644 changelogs/unreleased/security-fix-import-decompr-issue.yml create mode 100644 doc/security/project_import_decompressed_archive_size_limits.md create mode 100644 lib/gitlab/import_export/decompressed_archive_size_validator.rb create mode 100644 spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb mode change 100755 => 100644 vendor/gitignore/C++.gitignore mode change 100755 => 100644 vendor/gitignore/Java.gitignore diff --git a/changelogs/unreleased/security-fix-import-decompr-issue.yml b/changelogs/unreleased/security-fix-import-decompr-issue.yml new file mode 100644 index 00000000000..5f5a4bbc381 --- /dev/null +++ b/changelogs/unreleased/security-fix-import-decompr-issue.yml @@ -0,0 +1,5 @@ +--- +title: Add decompressed archive size validation on Project/Group Import +merge_request: 562 +author: +type: security diff --git a/doc/security/project_import_decompressed_archive_size_limits.md b/doc/security/project_import_decompressed_archive_size_limits.md new file mode 100644 index 00000000000..74b7a860c5b --- /dev/null +++ b/doc/security/project_import_decompressed_archive_size_limits.md @@ -0,0 +1,28 @@ +--- +type: reference, howto +--- + +# Project Import Decompressed Archive Size Limits + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31564) in GitLab 12.10. + +When using [Project Import](../user/project/settings/import_export.md), the size of the decompressed project archive is limited to 10Gb. + +If decompressed size exceeds this limit, `Decompressed archive size validation failed` error is returned. + +## Enable/disable size validation + +Decompressed size validation is enabled by default. +If you have a project with decompressed size exceeding this limit, +it is possible to disable the validation by turning off the +`validate_import_decompressed_archive_size` feature flag. + +Start a [Rails console](../administration/troubleshooting/debug.md#starting-a-rails-console-session). + +```ruby +# Disable +Feature.disable(:validate_import_decompressed_archive_size) + +# Enable +Feature.enable(:validate_import_decompressed_archive_size) +``` diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb new file mode 100644 index 00000000000..219821a7150 --- /dev/null +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require 'zlib' + +module Gitlab + module ImportExport + class DecompressedArchiveSizeValidator + include Gitlab::Utils::StrongMemoize + + DEFAULT_MAX_BYTES = 10.gigabytes.freeze + CHUNK_SIZE = 4096.freeze + + attr_reader :error + + def initialize(archive_path:, max_bytes: self.class.max_bytes) + @archive_path = archive_path + @max_bytes = max_bytes + @bytes_read = 0 + @total_reads = 0 + @denominator = 5 + @error = nil + end + + def valid? + strong_memoize(:valid) do + validate + end + end + + def self.max_bytes + DEFAULT_MAX_BYTES + end + + def archive_file + @archive_file ||= File.open(@archive_path) + end + + private + + def validate + until archive_file.eof? + compressed_chunk = archive_file.read(CHUNK_SIZE) + + inflate_stream.inflate(compressed_chunk) do |chunk| + @bytes_read += chunk.size + @total_reads += 1 + end + + # Start garbage collection every 5 reads in order + # to prevent memory bloat during archive decompression + GC.start if gc_start? + + if @bytes_read > @max_bytes + @error = error_message + + return false + end + end + + true + rescue => e + @error = error_message + + Gitlab::ErrorTracking.track_exception(e) + + Gitlab::Import::Logger.info( + message: @error, + error: e.message + ) + + false + ensure + inflate_stream.close + archive_file.close + end + + def inflate_stream + @inflate_stream ||= Zlib::Inflate.new(Zlib::MAX_WBITS + 32) + end + + def gc_start? + @total_reads % @denominator == 0 + end + + def error_message + _('Decompressed archive size validation failed.') + end + end + end +end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 9d04d55770d..3cb1eb72ceb 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -28,6 +28,7 @@ module Gitlab copy_archive wait_for_archived_file do + validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: true) decompress_archive end rescue => e @@ -82,6 +83,14 @@ module Gitlab def extracted_files Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) } end + + def validate_decompressed_archive_size + raise ImporterError.new(size_validator.error) unless size_validator.valid? + end + + def size_validator + @size_validator ||= DecompressedArchiveSizeValidator.new(archive_path: @archive_file) + end end end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index b41307b29fa..f2bed58438c 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -6540,6 +6540,9 @@ msgstr "" msgid "Decline and sign out" msgstr "" +msgid "Decompressed archive size validation failed." +msgstr "" + msgid "Default Branch" msgstr "" diff --git a/spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb b/spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb new file mode 100644 index 00000000000..302d11896c9 --- /dev/null +++ b/spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::DecompressedArchiveSizeValidator do + let_it_be(:filepath) { File.join(Dir.tmpdir, 'decompressed_archive_size_validator_spec.gz') } + + before(:all) do + create_compressed_file + end + + after(:all) do + FileUtils.rm(filepath) + end + + subject { described_class.new(archive_path: filepath, max_bytes: max_bytes) } + + describe '#valid?' do + let(:max_bytes) { 1 } + + context 'when file does not exceed allowed decompressed size' do + let(:max_bytes) { 20 } + + it 'returns true' do + expect(subject.valid?).to eq(true) + end + end + + context 'when file exceeds allowed decompressed size' do + it 'returns false' do + expect(subject.valid?).to eq(false) + end + end + + context 'when something goes wrong during decompression' do + before do + allow(subject.archive_file).to receive(:eof?).and_raise(StandardError) + end + + it 'logs and tracks raised exception' do + expect(Gitlab::ErrorTracking).to receive(:track_exception).with(instance_of(StandardError)) + expect(Gitlab::Import::Logger).to receive(:info).with(hash_including(message: 'Decompressed archive size validation failed.')) + + subject.valid? + end + + it 'returns false' do + expect(subject.valid?).to eq(false) + end + end + end + + def create_compressed_file + Zlib::GzipWriter.open(filepath) do |gz| + gz.write('Hello World!') + end + end +end diff --git a/spec/lib/gitlab/import_export/file_importer_spec.rb b/spec/lib/gitlab/import_export/file_importer_spec.rb index 7c54c5f2da1..870fe8bc7fc 100644 --- a/spec/lib/gitlab/import_export/file_importer_spec.rb +++ b/spec/lib/gitlab/import_export/file_importer_spec.rb @@ -98,6 +98,45 @@ describe Gitlab::ImportExport::FileImporter do end end + context 'when file exceeds acceptable decompressed size' do + let(:project) { create(:project) } + let(:shared) { Gitlab::ImportExport::Shared.new(project) } + let(:filepath) { File.join(Dir.tmpdir, 'file_importer_spec.gz') } + + subject { described_class.new(importable: project, archive_file: filepath, shared: shared) } + + before do + Zlib::GzipWriter.open(filepath) do |gz| + gz.write('Hello World!') + end + end + + context 'when validate_import_decompressed_archive_size feature flag is enabled' do + before do + stub_feature_flags(validate_import_decompressed_archive_size: true) + + allow(Gitlab::ImportExport::DecompressedArchiveSizeValidator).to receive(:max_bytes).and_return(1) + end + + it 'returns false' do + expect(subject.import).to eq(false) + expect(shared.errors.join).to eq('Decompressed archive size validation failed.') + end + end + + context 'when validate_import_decompressed_archive_size feature flag is disabled' do + before do + stub_feature_flags(validate_import_decompressed_archive_size: false) + end + + it 'skips validation' do + expect(subject).to receive(:validate_decompressed_archive_size).never + + subject.import + end + end + end + def setup_files FileUtils.mkdir_p("#{shared.export_path}/subfolder/") FileUtils.touch(valid_file) diff --git a/vendor/gitignore/C++.gitignore b/vendor/gitignore/C++.gitignore old mode 100755 new mode 100644 diff --git a/vendor/gitignore/Java.gitignore b/vendor/gitignore/Java.gitignore old mode 100755 new mode 100644 -- cgit v1.2.1