diff options
Diffstat (limited to 'lib/gitlab/import_export')
20 files changed, 52 insertions, 22 deletions
diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index 1e8009d29c2..78608a946de 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -32,6 +32,10 @@ module Gitlab end end + def delete_export? + false + end + private def send_file diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 959ece4b903..30cd5ccfbcb 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -69,6 +69,7 @@ module Gitlab # the relation_hash, updating references with new object IDs, mapping users using # the "members_mapper" object, also updating notes if required. def create + return @relation_hash if author_relation? return if invalid_relation? || predefined_relation? setup_base_models @@ -95,6 +96,10 @@ module Gitlab relation_class.try(:predefined_id?, @relation_hash['id']) end + def author_relation? + @relation_name == :author + end + def setup_models raise NotImplementedError end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index ace9d83dc9a..6c0b6de9e85 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -15,8 +15,17 @@ module Gitlab end def gzip(dir:, filename:) + gzip_with_options(dir: dir, filename: filename) + end + + def gunzip(dir:, filename:) + gzip_with_options(dir: dir, filename: filename, options: 'd') + end + + def gzip_with_options(dir:, filename:, options: nil) filepath = File.join(dir, filename) cmd = %W(gzip #{filepath}) + cmd << "-#{options}" if options _, status = Gitlab::Popen.popen(cmd) diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 2baf2c61f7c..febfe00af0b 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -32,7 +32,16 @@ module Gitlab Timeout.timeout(TIMEOUT_LIMIT) do stdin, stdout, stderr, wait_thr = Open3.popen3(command, pgroup: true) stdin.close - pgrp = Process.getpgid(wait_thr[:pid]) + + # When validation is performed on a small archive (e.g. 100 bytes) + # `wait_thr` finishes before we can get process group id. Do not + # raise exception in this scenario. + pgrp = begin + Process.getpgid(wait_thr[:pid]) + rescue Errno::ESRCH + nil + end + status = wait_thr.value if status.success? diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index 4af6b03fe94..af0026b8864 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -15,7 +15,7 @@ module Gitlab end def self.file_compression_error - self.new('File compression failed') + self.new('File compression/decompression failed') end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 4b3258f8caa..5274fcec43e 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -28,9 +28,7 @@ module Gitlab copy_archive wait_for_archived_file do - # Disable archive validation by default - # See: https://gitlab.com/gitlab-org/gitlab/-/issues/235949 - validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size) + validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: :yaml) decompress_archive end rescue StandardError => e diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index aceb4821a06..4786c7a52cc 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -70,11 +70,14 @@ ee: - :award_emoji - events: - :push_event_payload + - label_links: + - :label - notes: - :author - :award_emoji - events: - :push_event_payload + - :system_note_metadata - boards: - :board_assignee - :milestone diff --git a/lib/gitlab/import_export/group/legacy_import_export.yml b/lib/gitlab/import_export/group/legacy_import_export.yml index 19611e1b010..0a6234f9f02 100644 --- a/lib/gitlab/import_export/group/legacy_import_export.yml +++ b/lib/gitlab/import_export/group/legacy_import_export.yml @@ -72,6 +72,8 @@ ee: - :award_emoji - events: - :push_event_payload + - label_links: + - :label - notes: - :author - :award_emoji diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb index 2b95c098b59..8b39362b6bb 100644 --- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb +++ b/lib/gitlab/import_export/group/legacy_tree_restorer.rb @@ -55,11 +55,11 @@ module Gitlab def relation_reader strong_memoize(:relation_reader) do if @group_hash.present? - ImportExport::JSON::LegacyReader::Hash.new( + ImportExport::Json::LegacyReader::Hash.new( @group_hash, relation_names: reader.group_relation_names) else - ImportExport::JSON::LegacyReader::File.new( + ImportExport::Json::LegacyReader::File.new( File.join(shared.export_path, 'group.json'), relation_names: reader.group_relation_names) end diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index ea7de4cc896..19d707aaca5 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -118,7 +118,7 @@ module Gitlab def relation_reader strong_memoize(:relation_reader) do - ImportExport::JSON::NdjsonReader.new( + ImportExport::Json::NdjsonReader.new( File.join(shared.export_path, 'tree') ) end diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb index 0f588a55f9d..796b9258e57 100644 --- a/lib/gitlab/import_export/group/tree_saver.rb +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -42,7 +42,7 @@ module Gitlab end def serialize(group) - ImportExport::JSON::StreamingSerializer.new( + ImportExport::Json::StreamingSerializer.new( group, group_tree, json_writer, @@ -64,7 +64,7 @@ module Gitlab end def json_writer - @json_writer ||= ImportExport::JSON::NdjsonWriter.new(@full_path) + @json_writer ||= ImportExport::Json::NdjsonWriter.new(@full_path) end end end diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb index f29c0a44188..97b34088e3e 100644 --- a/lib/gitlab/import_export/json/legacy_reader.rb +++ b/lib/gitlab/import_export/json/legacy_reader.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class LegacyReader class File < LegacyReader include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/import_export/json/legacy_writer.rb b/lib/gitlab/import_export/json/legacy_writer.rb index 7be21410d26..e03ab9f7650 100644 --- a/lib/gitlab/import_export/json/legacy_writer.rb +++ b/lib/gitlab/import_export/json/legacy_writer.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class LegacyWriter include Gitlab::ImportExport::CommandLineUtil diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb index 5c8edd485e5..4899bd3b0ee 100644 --- a/lib/gitlab/import_export/json/ndjson_reader.rb +++ b/lib/gitlab/import_export/json/ndjson_reader.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class NdjsonReader MAX_JSON_DOCUMENT_SIZE = 50.megabytes diff --git a/lib/gitlab/import_export/json/ndjson_writer.rb b/lib/gitlab/import_export/json/ndjson_writer.rb index e74fdd74049..e303ac6eefa 100644 --- a/lib/gitlab/import_export/json/ndjson_writer.rb +++ b/lib/gitlab/import_export/json/ndjson_writer.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class NdjsonWriter include Gitlab::ImportExport::CommandLineUtil diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index ec42c5e51c0..d1e013a151c 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class StreamingSerializer include Gitlab::ImportExport::CommandLineUtil diff --git a/lib/gitlab/import_export/legacy_relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb index f8b8b74ffd7..c6b961ea210 100644 --- a/lib/gitlab/import_export/legacy_relation_tree_saver.rb +++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb @@ -22,7 +22,7 @@ module Gitlab private def batch_size(exportable) - Gitlab::ImportExport::JSON::StreamingSerializer.batch_size(exportable) + Gitlab::ImportExport::Json::StreamingSerializer.batch_size(exportable) end end end diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index 113502b4e3c..d8992061524 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -56,13 +56,13 @@ module Gitlab def ndjson_relation_reader return unless Feature.enabled?(:project_import_ndjson, project.namespace, default_enabled: true) - ImportExport::JSON::NdjsonReader.new( + ImportExport::Json::NdjsonReader.new( File.join(shared.export_path, 'tree') ) end def legacy_relation_reader - ImportExport::JSON::LegacyReader::File.new( + ImportExport::Json::LegacyReader::File.new( File.join(shared.export_path, 'project.json'), relation_names: reader.project_relation_names, allowed_path: importable_path diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 16012f3c0c0..1f0fa249390 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -14,7 +14,7 @@ module Gitlab end def save - ImportExport::JSON::StreamingSerializer.new( + ImportExport::Json::StreamingSerializer.new( exportable, reader.project_tree, json_writer, @@ -56,10 +56,10 @@ module Gitlab @json_writer ||= begin if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace, default_enabled: true) full_path = File.join(@shared.export_path, 'tree') - Gitlab::ImportExport::JSON::NdjsonWriter.new(full_path) + Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) else full_path = File.join(@shared.export_path, ImportExport.project_filename) - Gitlab::ImportExport::JSON::LegacyWriter.new(full_path, allowed_path: 'project') + Gitlab::ImportExport::Json::LegacyWriter.new(full_path, allowed_path: 'project') end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index f295ab38de0..5cb1c1f8981 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -88,7 +88,7 @@ module Gitlab when 'Project' @exportable.disk_path when 'Group' - @exportable.full_path + Storage::Hashed.new(@exportable, prefix: Storage::Hashed::GROUP_REPOSITORY_PATH_PREFIX).disk_path else raise Gitlab::ImportExport::Error, "Unsupported Exportable Type #{@exportable&.class}" end |