diff options
Diffstat (limited to 'lib/gitlab')
-rw-r--r-- | lib/gitlab/ci/config/entry/reports.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/ci/parsers.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/ci/parsers/coverage/cobertura.rb | 64 | ||||
-rw-r--r-- | lib/gitlab/ci/reports/coverage_reports.rb | 43 | ||||
-rw-r--r-- | lib/gitlab/import_export/group/tree_saver.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/json/legacy_writer.rb | 73 | ||||
-rw-r--r-- | lib/gitlab/import_export/json/streaming_serializer.rb | 82 | ||||
-rw-r--r-- | lib/gitlab/import_export/legacy_relation_tree_saver.rb (renamed from lib/gitlab/import_export/relation_tree_saver.rb) | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/legacy_tree_saver.rb | 68 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/tree_saver.rb | 46 |
10 files changed, 353 insertions, 33 deletions
diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 994d3799004..40d37f3601a 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management license_scanning metrics lsif - dotenv].freeze + dotenv cobertura].freeze attributes ALLOWED_KEYS @@ -35,6 +35,7 @@ module Gitlab validates :metrics, array_of_strings_or_string: true validates :lsif, array_of_strings_or_string: true validates :dotenv, array_of_strings_or_string: true + validates :cobertura, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index c76cd5ff285..a44105d53c2 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -9,7 +9,8 @@ module Gitlab def self.parsers { - junit: ::Gitlab::Ci::Parsers::Test::Junit + junit: ::Gitlab::Ci::Parsers::Test::Junit, + cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura } end diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb new file mode 100644 index 00000000000..006d5097148 --- /dev/null +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Coverage + class Cobertura + CoberturaParserError = Class.new(Gitlab::Ci::Parsers::ParserError) + + def parse!(xml_data, coverage_report) + root = Hash.from_xml(xml_data) + + parse_all(root, coverage_report) + rescue Nokogiri::XML::SyntaxError + raise CoberturaParserError, "XML parsing failed" + rescue + raise CoberturaParserError, "Cobertura parsing failed" + end + + private + + def parse_all(root, coverage_report) + return unless root.present? + + root.each do |key, value| + parse_node(key, value, coverage_report) + end + end + + def parse_node(key, value, coverage_report) + if key == 'class' + Array.wrap(value).each do |item| + parse_class(item, coverage_report) + end + elsif value.is_a?(Hash) + parse_all(value, coverage_report) + elsif value.is_a?(Array) + value.each do |item| + parse_all(item, coverage_report) + end + end + end + + def parse_class(file, coverage_report) + return unless file["filename"].present? && file["lines"].present? + + parsed_lines = parse_lines(file["lines"]) + + coverage_report.add_file(file["filename"], Hash[parsed_lines]) + end + + def parse_lines(lines) + line_array = Array.wrap(lines["line"]) + + line_array.map do |line| + # Using `Integer()` here to raise exception on invalid values + [Integer(line["number"]), Integer(line["hits"])] + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/coverage_reports.rb b/lib/gitlab/ci/reports/coverage_reports.rb new file mode 100644 index 00000000000..31afb636d2f --- /dev/null +++ b/lib/gitlab/ci/reports/coverage_reports.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class CoverageReports + attr_reader :files + + def initialize + @files = {} + end + + def pick(keys) + coverage_files = files.select do |key| + keys.include?(key) + end + + { files: coverage_files } + end + + def add_file(name, line_coverage) + if files[name].present? + line_coverage.each { |line, hits| combine_lines(name, line, hits) } + + else + files[name] = line_coverage + end + end + + private + + def combine_lines(name, line, hits) + if files[name][line].present? + files[name][line] += hits + + else + files[name][line] = hits + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb index 48f6925884b..fd1eb329ad2 100644 --- a/lib/gitlab/import_export/group/tree_saver.rb +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -49,7 +49,7 @@ module Gitlab end def tree_saver - @tree_saver ||= RelationTreeSaver.new + @tree_saver ||= LegacyRelationTreeSaver.new end end end diff --git a/lib/gitlab/import_export/json/legacy_writer.rb b/lib/gitlab/import_export/json/legacy_writer.rb new file mode 100644 index 00000000000..c935e360a65 --- /dev/null +++ b/lib/gitlab/import_export/json/legacy_writer.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module JSON + class LegacyWriter + include Gitlab::ImportExport::CommandLineUtil + + attr_reader :path + + def initialize(path) + @path = path + @last_array = nil + @keys = Set.new + + mkdir_p(File.dirname(@path)) + file.write('{}') + end + + def close + @file&.close + @file = nil + end + + def set(hash) + hash.each do |key, value| + write(key, value) + end + end + + def write(key, value) + raise ArgumentError, "key '#{key}' already written" if @keys.include?(key) + + # rewind by one byte, to overwrite '}' + file.pos = file.size - 1 + + file.write(',') if @keys.any? + file.write(key.to_json) + file.write(':') + file.write(value.to_json) + file.write('}') + + @keys.add(key) + @last_array = nil + @last_array_count = nil + end + + def append(key, value) + unless @last_array == key + write(key, []) + + @last_array = key + @last_array_count = 0 + end + + # rewind by two bytes, to overwrite ']}' + file.pos = file.size - 2 + + file.write(',') if @last_array_count > 0 + file.write(value.to_json) + file.write(']}') + @last_array_count += 1 + end + + private + + def file + @file ||= File.open(@path, "wb") + end + end + end + end +end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb new file mode 100644 index 00000000000..d053bf16166 --- /dev/null +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module JSON + class StreamingSerializer + include Gitlab::ImportExport::CommandLineUtil + + BATCH_SIZE = 100 + + class Raw < String + def to_json(*_args) + to_s + end + end + + def initialize(exportable, relations_schema, json_writer) + @exportable = exportable + @relations_schema = relations_schema + @json_writer = json_writer + end + + def execute + serialize_root + + includes.each do |relation_definition| + serialize_relation(relation_definition) + end + end + + private + + attr_reader :json_writer, :relations_schema, :exportable + + def serialize_root + attributes = exportable.as_json( + relations_schema.merge(include: nil, preloads: nil)) + json_writer.set(attributes) + end + + def serialize_relation(definition) + raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash) + raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one? + + key, options = definition.first + + record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend + if record.is_a?(ActiveRecord::Relation) + serialize_many_relations(key, record, options) + else + serialize_single_relation(key, record, options) + end + end + + def serialize_many_relations(key, records, options) + key_preloads = preloads&.dig(key) + records = records.preload(key_preloads) if key_preloads + + records.find_each(batch_size: BATCH_SIZE) do |record| + json = Raw.new(record.to_json(options)) + + json_writer.append(key, json) + end + end + + def serialize_single_relation(key, record, options) + json = Raw.new(record.to_json(options)) + + json_writer.write(key, json) + end + + def includes + relations_schema[:include] + end + + def preloads + relations_schema[:preload] + end + end + end + end +end diff --git a/lib/gitlab/import_export/relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb index ed5392c13d0..fe3e64358e5 100644 --- a/lib/gitlab/import_export/relation_tree_saver.rb +++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - class RelationTreeSaver + class LegacyRelationTreeSaver include Gitlab::ImportExport::CommandLineUtil def serialize(exportable, relations_tree) diff --git a/lib/gitlab/import_export/project/legacy_tree_saver.rb b/lib/gitlab/import_export/project/legacy_tree_saver.rb new file mode 100644 index 00000000000..2ed98f52c58 --- /dev/null +++ b/lib/gitlab/import_export/project/legacy_tree_saver.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class LegacyTreeSaver + attr_reader :full_path + + def initialize(project:, current_user:, shared:, params: {}) + @params = params + @project = project + @current_user = current_user + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) + end + + def save + project_tree = tree_saver.serialize(@project, reader.project_tree) + fix_project_tree(project_tree) + tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename) + + true + rescue => e + @shared.error(e) + false + end + + private + + # Aware that the resulting hash needs to be pure-hash and + # does not include any AR objects anymore, only objects that run `.to_json` + def fix_project_tree(project_tree) + if @params[:description].present? + project_tree['description'] = @params[:description] + end + + project_tree['project_members'] += group_members_array + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end + + def group_members_array + group_members.as_json(reader.group_members_tree).each do |group_member| + group_member['source_type'] = 'Project' # Make group members project members of the future import + end + end + + def group_members + return [] unless @current_user.can?(:admin_group, @project.group) + + # We need `.where.not(user_id: nil)` here otherwise when a group has an + # invitee, it would make the following query return 0 rows since a NULL + # user_id would be present in the subquery + # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values + non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id) + + GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) + end + + def tree_saver + @tree_saver ||= Gitlab::ImportExport::LegacyRelationTreeSaver.new + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 32b3b518ece..01000c9d6d9 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -15,52 +15,40 @@ module Gitlab end def save - project_tree = tree_saver.serialize(@project, reader.project_tree) - fix_project_tree(project_tree) - tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename) + json_writer = ImportExport::JSON::LegacyWriter.new(@full_path) + + serializer = ImportExport::JSON::StreamingSerializer.new(exportable, reader.project_tree, json_writer) + serializer.execute true rescue => e @shared.error(e) false + ensure + json_writer&.close end private - # Aware that the resulting hash needs to be pure-hash and - # does not include any AR objects anymore, only objects that run `.to_json` - def fix_project_tree(project_tree) - if @params[:description].present? - project_tree['description'] = @params[:description] - end - - project_tree['project_members'] += group_members_array - end - def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end - def group_members_array - group_members.as_json(reader.group_members_tree).each do |group_member| - group_member['source_type'] = 'Project' # Make group members project members of the future import - end + def exportable + @project.present(exportable_params) end - def group_members - return [] unless @current_user.can?(:admin_group, @project.group) - - # We need `.where.not(user_id: nil)` here otherwise when a group has an - # invitee, it would make the following query return 0 rows since a NULL - # user_id would be present in the subquery - # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values - non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id) - - GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) + def exportable_params + params = { + presenter_class: presenter_class, + current_user: @current_user + } + params[:override_description] = @params[:description] if @params[:description].present? + params end - def tree_saver - @tree_saver ||= RelationTreeSaver.new + def presenter_class + Projects::ImportExport::ProjectExportPresenter end end end |