diff options
Diffstat (limited to 'app/services/packages')
15 files changed, 192 insertions, 107 deletions
diff --git a/app/services/packages/debian/create_distribution_service.rb b/app/services/packages/debian/create_distribution_service.rb index b4b1ec952ef..218423bb8e3 100644 --- a/app/services/packages/debian/create_distribution_service.rb +++ b/app/services/packages/debian/create_distribution_service.rb @@ -61,7 +61,7 @@ module Packages create_objects(distribution.architectures, architectures, error_label: 'Architecture') end - def create_objects(objects, object_names_from_params, error_label: ) + def create_objects(objects, object_names_from_params, error_label:) object_names_from_params.each do |name| new_object = objects.create(name: name) append_errors(new_object, error_label) diff --git a/app/services/packages/debian/update_distribution_service.rb b/app/services/packages/debian/update_distribution_service.rb index 218167ecdc5..5096bd5361f 100644 --- a/app/services/packages/debian/update_distribution_service.rb +++ b/app/services/packages/debian/update_distribution_service.rb @@ -58,7 +58,7 @@ module Packages update_objects(distribution.architectures, architectures, error_label: 'Architecture') end - def update_objects(objects, object_names_from_params, error_label: ) + def update_objects(objects, object_names_from_params, error_label:) current_object_names = objects.map(&:name) missing_object_names = object_names_from_params - current_object_names extra_object_names = current_object_names - object_names_from_params diff --git a/app/services/packages/maven/metadata/base_create_xml_service.rb b/app/services/packages/maven/metadata/base_create_xml_service.rb index 4d5cab4978e..3b0d93e1dfb 100644 --- a/app/services/packages/maven/metadata/base_create_xml_service.rb +++ b/app/services/packages/maven/metadata/base_create_xml_service.rb @@ -8,13 +8,16 @@ module Packages INDENT_SPACE = 2 - def initialize(metadata_content:, package:) + def initialize(metadata_content:, package:, logger: nil) @metadata_content = metadata_content @package = package + @logger = logger || Gitlab::AppJsonLogger end private + attr_reader :logger + def xml_doc strong_memoize(:xml_doc) do Nokogiri::XML(@metadata_content) do |config| diff --git a/app/services/packages/maven/metadata/create_versions_xml_service.rb b/app/services/packages/maven/metadata/create_versions_xml_service.rb index 13b6efa8650..c2ac7fea703 100644 --- a/app/services/packages/maven/metadata/create_versions_xml_service.rb +++ b/app/services/packages/maven/metadata/create_versions_xml_service.rb @@ -67,6 +67,12 @@ module Packages def update_release return false if release_coherent? + unless release_xml_node.present? + log_malformed_content('Missing release tag') + + return false + end + if release_from_database release_xml_node.content = release_from_database else @@ -159,6 +165,15 @@ module Packages non_snapshot_versions_from_database.last end end + + def log_malformed_content(reason) + logger.warn( + message: 'A malformed metadata file has been encountered', + reason: reason, + project_id: @package.project_id, + package_id: @package.id + ) + end end end end diff --git a/app/services/packages/npm/create_package_service.rb b/app/services/packages/npm/create_package_service.rb index a3596314199..dd074f7472b 100644 --- a/app/services/packages/npm/create_package_service.rb +++ b/app/services/packages/npm/create_package_service.rb @@ -81,7 +81,7 @@ module Packages # - https://blog.aaronlenoir.com/2017/11/10/get-original-length-from-base-64-string/ # - https://en.wikipedia.org/wiki/Base64#Decoding_Base64_with_padding encoded_data = attachment['data'] - ((encoded_data.length * 0.75 ) - encoded_data[-2..].count('=')).to_i + ((encoded_data.length * 0.75) - encoded_data[-2..].count('=')).to_i end end diff --git a/app/services/packages/rpm/parse_package_service.rb b/app/services/packages/rpm/parse_package_service.rb index 689a161a81a..18b916a9d8b 100644 --- a/app/services/packages/rpm/parse_package_service.rb +++ b/app/services/packages/rpm/parse_package_service.rb @@ -25,7 +25,8 @@ module Packages epoch: package_tags[:epoch] || '0', changelogs: build_changelogs, requirements: build_requirements, - provides: build_provides + provides: build_provides, + directories: package_tags[:dirnames] }.merge(extract_static_attributes) end diff --git a/app/services/packages/rpm/repository_metadata/base_builder.rb b/app/services/packages/rpm/repository_metadata/base_builder.rb deleted file mode 100644 index 2c0a11457ec..00000000000 --- a/app/services/packages/rpm/repository_metadata/base_builder.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true -module Packages - module Rpm - module RepositoryMetadata - class BaseBuilder - def initialize(xml: nil, data: {}) - @xml = Nokogiri::XML(xml) if xml.present? - @data = data - end - - def execute - return build_empty_structure if xml.blank? - - update_xml_document - update_package_count - xml.to_xml - end - - private - - attr_reader :xml, :data - - def build_empty_structure - Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| - xml.method_missing(self.class::ROOT_TAG, self.class::ROOT_ATTRIBUTES) - end.to_xml - end - - def update_xml_document - # Add to the root xml element a new package metadata node - xml.at(self.class::ROOT_TAG).add_child(build_new_node) - end - - def update_package_count - packages_count = xml.css("//#{self.class::ROOT_TAG}/package").count - - xml.at(self.class::ROOT_TAG).attributes["packages"].value = packages_count.to_s - end - - def build_new_node - raise NotImplementedError, "#{self.class} should implement #{__method__}" - end - end - end - end -end diff --git a/app/services/packages/rpm/repository_metadata/build_filelist_xml.rb b/app/services/packages/rpm/repository_metadata/build_filelist_xml.rb deleted file mode 100644 index 01fb36f4b91..00000000000 --- a/app/services/packages/rpm/repository_metadata/build_filelist_xml.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true -module Packages - module Rpm - module RepositoryMetadata - class BuildFilelistXml < ::Packages::Rpm::RepositoryMetadata::BaseBuilder - ROOT_TAG = 'filelists' - ROOT_ATTRIBUTES = { - xmlns: 'http://linux.duke.edu/metadata/filelists', - packages: '0' - }.freeze - end - end - end -end diff --git a/app/services/packages/rpm/repository_metadata/build_filelist_xml_service.rb b/app/services/packages/rpm/repository_metadata/build_filelist_xml_service.rb new file mode 100644 index 00000000000..47cbba76fa4 --- /dev/null +++ b/app/services/packages/rpm/repository_metadata/build_filelist_xml_service.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true +module Packages + module Rpm + module RepositoryMetadata + class BuildFilelistXmlService < BuildXmlBaseService + ROOT_TAG = 'filelists' + ROOT_ATTRIBUTES = { + xmlns: 'http://linux.duke.edu/metadata/filelists', + packages: '0' + }.freeze + + def execute + super do |xml| + xml.package(pkgid: data[:pkgid], name: data[:name], arch: data[:arch]) do + xml.version epoch: data[:epoch], ver: data[:version], rel: data[:release] + build_file_nodes(xml) + end + end + end + + private + + def build_file_nodes(xml) + data[:files].each do |path| + attributes = dir?(path) ? { type: 'dir' } : {} + + xml.file path, **attributes + end + end + + def dir?(path) + # Add trailing slash to path to check + # if it exists in directories list + data[:directories].include? File.join(path, '') + end + end + end + end +end diff --git a/app/services/packages/rpm/repository_metadata/build_other_xml.rb b/app/services/packages/rpm/repository_metadata/build_other_xml.rb deleted file mode 100644 index 4bf61c901a3..00000000000 --- a/app/services/packages/rpm/repository_metadata/build_other_xml.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true -module Packages - module Rpm - module RepositoryMetadata - class BuildOtherXml < ::Packages::Rpm::RepositoryMetadata::BaseBuilder - ROOT_TAG = 'otherdata' - ROOT_ATTRIBUTES = { - xmlns: 'http://linux.duke.edu/metadata/other', - packages: '0' - }.freeze - end - end - end -end diff --git a/app/services/packages/rpm/repository_metadata/build_other_xml_service.rb b/app/services/packages/rpm/repository_metadata/build_other_xml_service.rb new file mode 100644 index 00000000000..00e88f4f548 --- /dev/null +++ b/app/services/packages/rpm/repository_metadata/build_other_xml_service.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true +module Packages + module Rpm + module RepositoryMetadata + class BuildOtherXmlService < BuildXmlBaseService + ROOT_TAG = 'otherdata' + ROOT_ATTRIBUTES = { + xmlns: 'http://linux.duke.edu/metadata/other', + packages: '0' + }.freeze + + def execute + super do |xml| + xml.package(pkgid: data[:pkgid], name: data[:name], arch: data[:arch]) do + xml.version epoch: data[:epoch], ver: data[:version], rel: data[:release] + build_changelog_nodes(xml) + end + end + end + + private + + def build_changelog_nodes(xml) + data[:changelogs].each do |changelog| + xml.changelog changelog[:changelogtext], date: changelog[:changelogtime] + end + end + end + end + end +end diff --git a/app/services/packages/rpm/repository_metadata/build_primary_xml.rb b/app/services/packages/rpm/repository_metadata/build_primary_xml_service.rb index 580bf844a0c..1044ab3997a 100644 --- a/app/services/packages/rpm/repository_metadata/build_primary_xml.rb +++ b/app/services/packages/rpm/repository_metadata/build_primary_xml_service.rb @@ -2,7 +2,7 @@ module Packages module Rpm module RepositoryMetadata - class BuildPrimaryXml < ::Packages::Rpm::RepositoryMetadata::BaseBuilder + class BuildPrimaryXmlService < BuildXmlBaseService ROOT_TAG = 'metadata' ROOT_ATTRIBUTES = { xmlns: 'http://linux.duke.edu/metadata/common', @@ -11,37 +11,27 @@ module Packages }.freeze # Nodes that have only text without attributes - REQUIRED_BASE_ATTRIBUTES = %i[name arch summary description].freeze - NOT_REQUIRED_BASE_ATTRIBUTES = %i[url packager].freeze + BASE_ATTRIBUTES = %i[name arch summary description url packager].freeze FORMAT_NODE_BASE_ATTRIBUTES = %i[license vendor group buildhost sourcerpm].freeze - private - - def build_new_node - builder = Nokogiri::XML::Builder.new do |xml| + def execute + super do |xml| xml.package(type: :rpm, 'xmlns:rpm': 'http://linux.duke.edu/metadata/rpm') do - build_required_base_attributes(xml) - build_not_required_base_attributes(xml) + build_base_attributes(xml) xml.version epoch: data[:epoch], ver: data[:version], rel: data[:release] - xml.checksum data[:checksum], type: 'sha256', pkgid: 'YES' + xml.checksum data[:pkgid], type: 'sha256', pkgid: 'YES' xml.size package: data[:packagesize], installed: data[:installedsize], archive: data[:archivesize] xml.time file: data[:filetime], build: data[:buildtime] xml.location href: data[:location] if data[:location].present? build_format_node(xml) end end - - Nokogiri::XML(builder.to_xml).at('package') end - def build_required_base_attributes(xml) - REQUIRED_BASE_ATTRIBUTES.each do |attribute| - xml.method_missing(attribute, data[attribute]) - end - end + private - def build_not_required_base_attributes(xml) - NOT_REQUIRED_BASE_ATTRIBUTES.each do |attribute| + def build_base_attributes(xml) + BASE_ATTRIBUTES.each do |attribute| xml.method_missing(attribute, data[attribute]) if data[attribute].present? end end diff --git a/app/services/packages/rpm/repository_metadata/build_repomd_xml.rb b/app/services/packages/rpm/repository_metadata/build_repomd_xml_service.rb index 84614196254..cb80faa12c0 100644 --- a/app/services/packages/rpm/repository_metadata/build_repomd_xml.rb +++ b/app/services/packages/rpm/repository_metadata/build_repomd_xml_service.rb @@ -2,9 +2,7 @@ module Packages module Rpm module RepositoryMetadata - class BuildRepomdXml - attr_reader :data - + class BuildRepomdXmlService ROOT_ATTRIBUTES = { xmlns: 'http://linux.duke.edu/metadata/repo', 'xmlns:rpm': 'http://linux.duke.edu/metadata/rpm' @@ -26,12 +24,6 @@ module Packages end def execute - build_repomd - end - - private - - def build_repomd Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| xml.repomd(ROOT_ATTRIBUTES) do xml.revision Time.now.to_i @@ -40,6 +32,10 @@ module Packages end.to_xml end + private + + attr_reader :data + def build_data_info(xml) data.each do |filename, info| xml.data(type: filename) do diff --git a/app/services/packages/rpm/repository_metadata/build_xml_base_service.rb b/app/services/packages/rpm/repository_metadata/build_xml_base_service.rb new file mode 100644 index 00000000000..4dfb4087f1b --- /dev/null +++ b/app/services/packages/rpm/repository_metadata/build_xml_base_service.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +module Packages + module Rpm + module RepositoryMetadata + class BuildXmlBaseService + def initialize(data) + @data = data + end + + def execute + builder = Nokogiri::XML::Builder.new { |xml| yield xml } # rubocop:disable Style/ExplicitBlockArgument + + Nokogiri::XML(builder.to_xml).at('package') + end + + private + + attr_reader :data + end + end + end +end diff --git a/app/services/packages/rpm/repository_metadata/update_xml_service.rb b/app/services/packages/rpm/repository_metadata/update_xml_service.rb new file mode 100644 index 00000000000..8fef425195f --- /dev/null +++ b/app/services/packages/rpm/repository_metadata/update_xml_service.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true +module Packages + module Rpm + module RepositoryMetadata + class UpdateXmlService + BUILDERS = { + other: ::Packages::Rpm::RepositoryMetadata::BuildOtherXmlService, + primary: ::Packages::Rpm::RepositoryMetadata::BuildPrimaryXmlService, + filelist: ::Packages::Rpm::RepositoryMetadata::BuildFilelistXmlService + }.freeze + + def initialize(filename:, xml: nil, data: {}) + @builder_class = BUILDERS[filename] + raise ArgumentError, "Filename must be one of: #{BUILDERS.keys.join(', ')}" if @builder_class.nil? + + @xml = Nokogiri::XML(xml) if xml.present? + @data = data + @filename = filename + end + + def execute + return build_empty_structure if xml.blank? + + remove_existing_packages + update_xml_document + update_package_count + xml.to_xml + end + + private + + attr_reader :xml, :data, :builder_class, :filename + + def build_empty_structure + Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| + xml.method_missing(builder_class::ROOT_TAG, builder_class::ROOT_ATTRIBUTES) + end.to_xml + end + + def update_xml_document + # Add to the root xml element a new package metadata node + xml.at(builder_class::ROOT_TAG).add_child(builder_class.new(data).execute) + end + + def update_package_count + packages_count = xml.css("//#{builder_class::ROOT_TAG}/package").count + + xml.at(builder_class::ROOT_TAG).attributes["packages"].value = packages_count.to_s + end + + def remove_existing_packages + case filename + when :primary + xml.search("checksum:contains('#{data[:pkgid]}')").each { _1.parent&.remove } + else + xml.search("[pkgid='#{data[:pkgid]}']").each(&:remove) + end + end + end + end + end +end |