summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-08-10 13:31:48 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2020-08-10 13:31:48 +0000
commit0a3f1f55493660acfabd198d2e1649a881d8852b (patch)
tree3edcd287de2cd0f7e72917865a5e5c4111b74ecb
parentea4766228b5536c83f1917d6058be913472ffa2d (diff)
downloadgitlab-ce-0a3f1f55493660acfabd198d2e1649a881d8852b.tar.gz
Add latest changes from gitlab-org/gitlab@13-2-stable-ee
-rw-r--r--changelogs/unreleased/security-fix-import-decompr-issue.yml5
-rw-r--r--doc/security/project_import_decompressed_archive_size_limits.md28
-rw-r--r--lib/gitlab/import_export/decompressed_archive_size_validator.rb90
-rw-r--r--lib/gitlab/import_export/file_importer.rb9
-rw-r--r--locale/gitlab.pot3
-rw-r--r--spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb58
-rw-r--r--spec/lib/gitlab/import_export/file_importer_spec.rb39
7 files changed, 232 insertions, 0 deletions
diff --git a/changelogs/unreleased/security-fix-import-decompr-issue.yml b/changelogs/unreleased/security-fix-import-decompr-issue.yml
new file mode 100644
index 00000000000..bda41369e65
--- /dev/null
+++ b/changelogs/unreleased/security-fix-import-decompr-issue.yml
@@ -0,0 +1,5 @@
+---
+title: Add decompressed archive size validation on Project/Group Import
+merge_request: 38736
+author:
+type: security
diff --git a/doc/security/project_import_decompressed_archive_size_limits.md b/doc/security/project_import_decompressed_archive_size_limits.md
new file mode 100644
index 00000000000..dd67db23d6b
--- /dev/null
+++ b/doc/security/project_import_decompressed_archive_size_limits.md
@@ -0,0 +1,28 @@
+---
+type: reference, howto
+---
+
+# Project Import Decompressed Archive Size Limits
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31564) in GitLab 13.2.
+
+When using [Project Import](../user/project/settings/import_export.md), the size of the decompressed project archive is limited to 10Gb.
+
+If decompressed size exceeds this limit, `Decompressed archive size validation failed` error is returned.
+
+## Enable/disable size validation
+
+Decompressed size validation is enabled by default.
+If you have a project with decompressed size exceeding this limit,
+it is possible to disable the validation by turning off the
+`validate_import_decompressed_archive_size` feature flag.
+
+Start a [Rails console](../administration/troubleshooting/debug.md#starting-a-rails-console-session).
+
+```ruby
+# Disable
+Feature.disable(:validate_import_decompressed_archive_size)
+
+# Enable
+Feature.enable(:validate_import_decompressed_archive_size)
+```
diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb
new file mode 100644
index 00000000000..219821a7150
--- /dev/null
+++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb
@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+module Gitlab
+ module ImportExport
+ class DecompressedArchiveSizeValidator
+ include Gitlab::Utils::StrongMemoize
+
+ DEFAULT_MAX_BYTES = 10.gigabytes.freeze
+ CHUNK_SIZE = 4096.freeze
+
+ attr_reader :error
+
+ def initialize(archive_path:, max_bytes: self.class.max_bytes)
+ @archive_path = archive_path
+ @max_bytes = max_bytes
+ @bytes_read = 0
+ @total_reads = 0
+ @denominator = 5
+ @error = nil
+ end
+
+ def valid?
+ strong_memoize(:valid) do
+ validate
+ end
+ end
+
+ def self.max_bytes
+ DEFAULT_MAX_BYTES
+ end
+
+ def archive_file
+ @archive_file ||= File.open(@archive_path)
+ end
+
+ private
+
+ def validate
+ until archive_file.eof?
+ compressed_chunk = archive_file.read(CHUNK_SIZE)
+
+ inflate_stream.inflate(compressed_chunk) do |chunk|
+ @bytes_read += chunk.size
+ @total_reads += 1
+ end
+
+ # Start garbage collection every 5 reads in order
+ # to prevent memory bloat during archive decompression
+ GC.start if gc_start?
+
+ if @bytes_read > @max_bytes
+ @error = error_message
+
+ return false
+ end
+ end
+
+ true
+ rescue => e
+ @error = error_message
+
+ Gitlab::ErrorTracking.track_exception(e)
+
+ Gitlab::Import::Logger.info(
+ message: @error,
+ error: e.message
+ )
+
+ false
+ ensure
+ inflate_stream.close
+ archive_file.close
+ end
+
+ def inflate_stream
+ @inflate_stream ||= Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
+ end
+
+ def gc_start?
+ @total_reads % @denominator == 0
+ end
+
+ def error_message
+ _('Decompressed archive size validation failed.')
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 9d04d55770d..3cb1eb72ceb 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -28,6 +28,7 @@ module Gitlab
copy_archive
wait_for_archived_file do
+ validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: true)
decompress_archive
end
rescue => e
@@ -82,6 +83,14 @@ module Gitlab
def extracted_files
Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) }
end
+
+ def validate_decompressed_archive_size
+ raise ImporterError.new(size_validator.error) unless size_validator.valid?
+ end
+
+ def size_validator
+ @size_validator ||= DecompressedArchiveSizeValidator.new(archive_path: @archive_file)
+ end
end
end
end
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index f983357d037..5f6d245aab6 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -7495,6 +7495,9 @@ msgstr ""
msgid "Decline and sign out"
msgstr ""
+msgid "Decompressed archive size validation failed."
+msgstr ""
+
msgid "Default Branch"
msgstr ""
diff --git a/spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb b/spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb
new file mode 100644
index 00000000000..efb271086a0
--- /dev/null
+++ b/spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::ImportExport::DecompressedArchiveSizeValidator do
+ let_it_be(:filepath) { File.join(Dir.tmpdir, 'decompressed_archive_size_validator_spec.gz') }
+
+ before(:all) do
+ create_compressed_file
+ end
+
+ after(:all) do
+ FileUtils.rm(filepath)
+ end
+
+ subject { described_class.new(archive_path: filepath, max_bytes: max_bytes) }
+
+ describe '#valid?' do
+ let(:max_bytes) { 1 }
+
+ context 'when file does not exceed allowed decompressed size' do
+ let(:max_bytes) { 20 }
+
+ it 'returns true' do
+ expect(subject.valid?).to eq(true)
+ end
+ end
+
+ context 'when file exceeds allowed decompressed size' do
+ it 'returns false' do
+ expect(subject.valid?).to eq(false)
+ end
+ end
+
+ context 'when something goes wrong during decompression' do
+ before do
+ allow(subject.archive_file).to receive(:eof?).and_raise(StandardError)
+ end
+
+ it 'logs and tracks raised exception' do
+ expect(Gitlab::ErrorTracking).to receive(:track_exception).with(instance_of(StandardError))
+ expect(Gitlab::Import::Logger).to receive(:info).with(hash_including(message: 'Decompressed archive size validation failed.'))
+
+ subject.valid?
+ end
+
+ it 'returns false' do
+ expect(subject.valid?).to eq(false)
+ end
+ end
+ end
+
+ def create_compressed_file
+ Zlib::GzipWriter.open(filepath) do |gz|
+ gz.write('Hello World!')
+ end
+ end
+end
diff --git a/spec/lib/gitlab/import_export/file_importer_spec.rb b/spec/lib/gitlab/import_export/file_importer_spec.rb
index 47485cc7edb..dc668e972cf 100644
--- a/spec/lib/gitlab/import_export/file_importer_spec.rb
+++ b/spec/lib/gitlab/import_export/file_importer_spec.rb
@@ -98,6 +98,45 @@ RSpec.describe Gitlab::ImportExport::FileImporter do
end
end
+ context 'when file exceeds acceptable decompressed size' do
+ let(:project) { create(:project) }
+ let(:shared) { Gitlab::ImportExport::Shared.new(project) }
+ let(:filepath) { File.join(Dir.tmpdir, 'file_importer_spec.gz') }
+
+ subject { described_class.new(importable: project, archive_file: filepath, shared: shared) }
+
+ before do
+ Zlib::GzipWriter.open(filepath) do |gz|
+ gz.write('Hello World!')
+ end
+ end
+
+ context 'when validate_import_decompressed_archive_size feature flag is enabled' do
+ before do
+ stub_feature_flags(validate_import_decompressed_archive_size: true)
+
+ allow(Gitlab::ImportExport::DecompressedArchiveSizeValidator).to receive(:max_bytes).and_return(1)
+ end
+
+ it 'returns false' do
+ expect(subject.import).to eq(false)
+ expect(shared.errors.join).to eq('Decompressed archive size validation failed.')
+ end
+ end
+
+ context 'when validate_import_decompressed_archive_size feature flag is disabled' do
+ before do
+ stub_feature_flags(validate_import_decompressed_archive_size: false)
+ end
+
+ it 'skips validation' do
+ expect(subject).to receive(:validate_decompressed_archive_size).never
+
+ subject.import
+ end
+ end
+ end
+
def setup_files
FileUtils.mkdir_p("#{shared.export_path}/subfolder/")
FileUtils.touch(valid_file)