Add latest changes from gitlab-org/gitlab@13-8-stable-eev13.8.0-rc42

6 files changed, 244 insertions, 37 deletions

diff --git a/app/services/packages/create_event_service.rb b/app/services/packages/create_event_service.rb
index f0328ceb08a..63248ef07c9 100644
--- a/app/services/packages/create_event_service.rb
+++ b/app/services/packages/create_event_service.rb
@@ -3,11 +3,13 @@
 module Packages
   class CreateEventService < BaseService
     def execute
-      if Feature.enabled?(:collect_package_events_redis) && redis_event_name
-        if guest?
-          ::Gitlab::UsageDataCounters::GuestPackageEventCounter.count(redis_event_name)
-        else
-          ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event(current_user.id, redis_event_name)
+      if Feature.enabled?(:collect_package_events_redis, default_enabled: true)
+        ::Packages::Event.unique_counters_for(event_scope, event_name, originator_type).each do |event_name|
+          ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: current_user.id)
+        end
+
+        ::Packages::Event.counters_for(event_scope, event_name, originator_type).each do |event_name|
+          ::Gitlab::UsageDataCounters::PackageEventCounter.count(event_name)
         end
       end
 
@@ -23,10 +25,6 @@ module Packages
 
     private
 
-    def redis_event_name
-      @redis_event_name ||= ::Packages::Event.allowed_event_name(event_scope, event_name, originator_type)
-    end
-
     def event_scope
       @event_scope ||= scope.is_a?(::Packages::Package) ? scope.package_type : scope
     end
diff --git a/app/services/packages/debian/create_package_file_service.rb b/app/services/packages/debian/create_package_file_service.rb
new file mode 100644
index 00000000000..2022a63a725
--- /dev/null
+++ b/app/services/packages/debian/create_package_file_service.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Packages
+  module Debian
+    class CreatePackageFileService
+      def initialize(package, params)
+        @package = package
+        @params = params
+      end
+
+      def execute
+        raise ArgumentError, "Invalid package" unless package.present?
+
+        # Debian package file are first uploaded to incoming with empty metadata,
+        # and are moved later by Packages::Debian::ProcessChangesService
+        package.package_files.create!(
+          file:        params[:file],
+          size:        params[:file]&.size,
+          file_name:   params[:file_name],
+          file_sha1:   params[:file_sha1],
+          file_sha256: params[:file]&.sha256,
+          file_md5:    params[:file_md5],
+          debian_file_metadatum_attributes: {
+            file_type: 'unknown',
+            architecture: nil,
+            fields: nil
+          }
+        )
+      end
+
+      private
+
+      attr_reader :package, :params
+    end
+  end
+end
diff --git a/app/services/packages/debian/extract_metadata_service.rb b/app/services/packages/debian/extract_metadata_service.rb
new file mode 100644
index 00000000000..fd5832bc0ba
--- /dev/null
+++ b/app/services/packages/debian/extract_metadata_service.rb
@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Packages
+  module Debian
+    class ExtractMetadataService
+      include Gitlab::Utils::StrongMemoize
+
+      ExtractionError = Class.new(StandardError)
+
+      def initialize(package_file)
+        @package_file = package_file
+      end
+
+      def execute
+        raise ExtractionError.new('invalid package file') unless valid_package_file?
+
+        extract_metadata
+      end
+
+      private
+
+      attr_reader :package_file
+
+      def valid_package_file?
+        package_file &&
+          package_file.package&.debian? &&
+          package_file.file.size > 0 # rubocop:disable Style/ZeroLengthPredicate
+      end
+
+      def file_type_basic
+        %i[dsc deb udeb buildinfo changes].each do |format|
+          return format if package_file.file_name.end_with?(".#{format}")
+        end
+
+        nil
+      end
+
+      def file_type_source
+        # https://manpages.debian.org/buster/dpkg-dev/dpkg-source.1.en.html
+        %i[gzip bzip2 lzma xz].each do |format|
+          return :source if package_file.file_name.end_with?(".tar.#{format}")
+        end
+
+        nil
+      end
+
+      def file_type
+        strong_memoize(:file_type) do
+          file_type_basic || file_type_source || :unknown
+        end
+      end
+
+      def file_type_debian?
+        file_type == :deb || file_type == :udeb
+      end
+
+      def file_type_meta?
+        file_type == :dsc || file_type == :buildinfo || file_type == :changes
+      end
+
+      def extracted_fields
+        if file_type_debian?
+          package_file.file.use_file do |file_path|
+            ::Packages::Debian::ExtractDebMetadataService.new(file_path).execute
+          end
+        elsif file_type_meta?
+          package_file.file.use_file do |file_path|
+            ::Packages::Debian::ParseDebian822Service.new(File.read(file_path)).execute.each_value.first
+          end
+        end
+      end
+
+      def extract_metadata
+        fields = extracted_fields
+        architecture = fields.delete(:Architecture) if file_type_debian?
+
+        {
+          file_type: file_type,
+          architecture: architecture,
+          fields: fields
+        }
+      end
+    end
+  end
+end
diff --git a/app/services/packages/debian/get_or_create_incoming_service.rb b/app/services/packages/debian/get_or_create_incoming_service.rb
new file mode 100644
index 00000000000..09e7877a2b4
--- /dev/null
+++ b/app/services/packages/debian/get_or_create_incoming_service.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Packages
+  module Debian
+    class GetOrCreateIncomingService < ::Packages::CreatePackageService
+      def execute
+        find_or_create_package!(:debian, name: 'incoming', version: nil)
+      end
+    end
+  end
+end
diff --git a/app/services/packages/maven/find_or_create_package_service.rb b/app/services/packages/maven/find_or_create_package_service.rb
index f598b5e7cd4..8ee449cbfdc 100644
--- a/app/services/packages/maven/find_or_create_package_service.rb
+++ b/app/services/packages/maven/find_or_create_package_service.rb
@@ -2,14 +2,23 @@
 module Packages
   module Maven
     class FindOrCreatePackageService < BaseService
-      MAVEN_METADATA_FILE = 'maven-metadata.xml'.freeze
-      SNAPSHOT_TERM = '-SNAPSHOT'.freeze
+      MAVEN_METADATA_FILE = 'maven-metadata.xml'
+      SNAPSHOT_TERM = '-SNAPSHOT'
 
       def execute
         package =
           ::Packages::Maven::PackageFinder.new(params[:path], current_user, project: project)
                                           .execute
 
+        unless Namespace::PackageSetting.duplicates_allowed?(package)
+          files = package&.package_files || []
+          current_maven_files = files.map { |file| extname(file.file_name) }
+
+          if current_maven_files.compact.include?(extname(params[:file_name]))
+            return ServiceResponse.error(message: 'Duplicate package is not allowed')
+          end
+        end
+
         unless package
           # Maven uploads several files during `mvn deploy` in next order:
           #   - my-company/my-app/1.0-SNAPSHOT/my-app.jar
@@ -48,7 +57,15 @@ module Packages
 
         package.build_infos.safe_find_or_create_by!(pipeline: params[:build].pipeline) if params[:build].present?
 
-        package
+        ServiceResponse.success(payload: { package: package })
+      end
+
+      private
+
+      def extname(filename)
+        return if filename.blank?
+
+        File.extname(filename)
       end
     end
   end
diff --git a/app/services/packages/nuget/search_service.rb b/app/services/packages/nuget/search_service.rb
index b95aa30bec1..1eead1e62b3 100644
--- a/app/services/packages/nuget/search_service.rb
+++ b/app/services/packages/nuget/search_service.rb
@@ -3,6 +3,7 @@
 module Packages
   module Nuget
     class SearchService < BaseService
+      include ::Packages::FinderHelper
       include Gitlab::Utils::StrongMemoize
       include ActiveRecord::ConnectionAdapters::Quoting
 
@@ -16,8 +17,9 @@ module Packages
         padding: 0
       }.freeze
 
-      def initialize(project, search_term, options = {})
-        @project = project
+      def initialize(current_user, project_or_group, search_term, options = {})
+        @current_user = current_user
+        @project_or_group = project_or_group
         @search_term = search_term
         @options = DEFAULT_OPTIONS.merge(options)
 
@@ -26,8 +28,8 @@ module Packages
       end
 
       def execute
-        OpenStruct.new(
-          total_count: package_names.total_count,
+        Result.new(
+          total_count: non_paginated_matching_package_names.count,
           results: search_packages
         )
       end
@@ -39,52 +41,104 @@ module Packages
         # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24182#technical-notes
         # and https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource
         subquery_name = :partition_subquery
-        arel_table = Arel::Table.new(:partition_subquery)
+        arel_table = Arel::Table.new(subquery_name)
         column_names = Packages::Package.column_names.map do |cn|
           "#{subquery_name}.#{quote_column_name(cn)}"
         end
 
         # rubocop: disable CodeReuse/ActiveRecord
-        pkgs = Packages::Package.select(column_names.join(','))
-                                .from(package_names_partition, subquery_name)
-                                .where(arel_table[:row_number].lteq(MAX_VERSIONS_PER_PACKAGE))
+        pkgs = Packages::Package
+        pkgs = pkgs.with(project_ids_cte.to_arel) if use_project_ids_cte?
+        pkgs = pkgs.select(column_names.join(','))
+                   .from(package_names_partition, subquery_name)
+                   .where(arel_table[:row_number].lteq(MAX_VERSIONS_PER_PACKAGE))
 
         return pkgs if include_prerelease_versions?
 
         # we can't use pkgs.without_version_like since we have a custom from
         pkgs.where.not(arel_table[:version].matches(PRE_RELEASE_VERSION_MATCHING_TERM))
+        # rubocop: enable CodeReuse/ActiveRecord
       end
 
       def package_names_partition
+        # rubocop: disable CodeReuse/ActiveRecord
         table_name = quote_table_name(Packages::Package.table_name)
         name_column = "#{table_name}.#{quote_column_name('name')}"
         created_at_column = "#{table_name}.#{quote_column_name('created_at')}"
         select_sql = "ROW_NUMBER() OVER (PARTITION BY #{name_column} ORDER BY #{created_at_column} DESC) AS row_number, #{table_name}.*"
 
-        @project.packages
-                .select(select_sql)
-                .nuget
-                .has_version
-                .without_nuget_temporary_name
-                .with_name(package_names)
+        nuget_packages.select(select_sql)
+                      .with_name(paginated_matching_package_names)
+                      .where(project_id: project_ids)
+        # rubocop: enable CodeReuse/ActiveRecord
       end
 
-      def package_names
-        strong_memoize(:package_names) do
-          pkgs = @project.packages
-                         .nuget
-                         .has_version
-                         .without_nuget_temporary_name
-                         .order_name
-                         .select_distinct_name
+      def paginated_matching_package_names
+        pkgs = base_matching_package_names
+        pkgs.page(0) # we're using a padding
+            .per(per_page)
+            .padding(padding)
+      end
+
+      def non_paginated_matching_package_names
+        # rubocop: disable CodeReuse/ActiveRecord
+        pkgs = base_matching_package_names
+        pkgs = pkgs.with(project_ids_cte.to_arel) if use_project_ids_cte?
+        pkgs
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+
+      def base_matching_package_names
+        strong_memoize(:base_matching_package_names) do
+          # rubocop: disable CodeReuse/ActiveRecord
+          pkgs = nuget_packages.order_name
+                               .select_distinct_name
+                               .where(project_id: project_ids)
           pkgs = pkgs.without_version_like(PRE_RELEASE_VERSION_MATCHING_TERM) unless include_prerelease_versions?
           pkgs = pkgs.search_by_name(@search_term) if @search_term.present?
-          pkgs.page(0) # we're using a padding
-              .per(per_page)
-              .padding(padding)
+          pkgs
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
 
+      def nuget_packages
+        Packages::Package.nuget
+                         .has_version
+                         .without_nuget_temporary_name
+      end
+
+      def project_ids_cte
+        return unless use_project_ids_cte?
+
+        strong_memoize(:project_ids_cte) do
+          query = projects_visible_to_user(@current_user, within_group: @project_or_group)
+          Gitlab::SQL::CTE.new(:project_ids, query.select(:id))
+        end
+      end
+
+      def project_ids
+        return @project_or_group.id if project?
+
+        if use_project_ids_cte?
+          # rubocop: disable CodeReuse/ActiveRecord
+          Project.select(:id)
+                 .from(project_ids_cte.table)
+          # rubocop: enable CodeReuse/ActiveRecord
+        end
+      end
+
+      def use_project_ids_cte?
+        group?
+      end
+
+      def project?
+        @project_or_group.is_a?(::Project)
+      end
+
+      def group?
+        @project_or_group.is_a?(::Group)
+      end
+
       def include_prerelease_versions?
         @options[:include_prerelease_versions]
       end
@@ -96,6 +150,12 @@ module Packages
       def per_page
         [@options[:per_page], MAX_PER_PAGE].min
       end
+
+      class Result
+        include ActiveModel::Model
+
+        attr_accessor :results, :total_count
+      end
     end
   end
 end