Add latest changes from gitlab-org/gitlab@master

12 files changed, 133 insertions, 93 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index ba77203154e..b37751e1b47 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -194,7 +194,8 @@ module API
       mount ::API::NugetPackages
       mount ::API::PypiPackages
       mount ::API::ComposerPackages
-      mount ::API::ConanPackages
+      mount ::API::ConanProjectPackages
+      mount ::API::ConanInstancePackages
       mount ::API::MavenPackages
       mount ::API::NpmPackages
       mount ::API::GenericPackages
diff --git a/lib/api/conan_instance_packages.rb b/lib/api/conan_instance_packages.rb
new file mode 100644
index 00000000000..209748d79fa
--- /dev/null
+++ b/lib/api/conan_instance_packages.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Conan Instance-Level Package Manager Client API
+module API
+  class ConanInstancePackages < Grape::API::Instance
+    namespace 'packages/conan/v1' do
+      include ConanPackageEndpoints
+    end
+  end
+end
diff --git a/lib/api/conan_packages.rb b/lib/api/conan_package_endpoints.rb
index 899683de184..445447cfcd2 100644
--- a/lib/api/conan_packages.rb
+++ b/lib/api/conan_package_endpoints.rb
@@ -9,8 +9,8 @@
 #
 # Technical debt: https://gitlab.com/gitlab-org/gitlab/issues/35798
 module API
-  class ConanPackages < Grape::API::Instance
-    helpers ::API::Helpers::PackagesManagerClientsHelpers
+  module ConanPackageEndpoints
+    extend ActiveSupport::Concern
 
     PACKAGE_REQUIREMENTS = {
       package_name: API::NO_SLASH_URL_PART_REGEX,
@@ -28,15 +28,19 @@ module API
 
     CONAN_FILES = (Gitlab::Regex::Packages::CONAN_RECIPE_FILES + Gitlab::Regex::Packages::CONAN_PACKAGE_FILES).freeze
 
-    before do
-      require_packages_enabled!
+    included do
+      helpers ::API::Helpers::PackagesManagerClientsHelpers
+      helpers ::API::Helpers::Packages::Conan::ApiHelpers
+      helpers ::API::Helpers::RelatedResourcesHelpers
 
-      # Personal access token will be extracted from Bearer or Basic authorization
-      # in the overridden find_personal_access_token or find_user_from_job_token helpers
-      authenticate!
-    end
+      before do
+        require_packages_enabled!
+
+        # Personal access token will be extracted from Bearer or Basic authorization
+        # in the overridden find_personal_access_token or find_user_from_job_token helpers
+        authenticate!
+      end
 
-    namespace 'packages/conan/v1' do
       desc 'Ping the Conan API' do
         detail 'This feature was introduced in GitLab 12.2'
       end
@@ -242,7 +246,7 @@ module API
         delete do
           authorize!(:destroy_package, project)
 
-          package_event('delete_package')
+          package_event('delete_package', category: 'API::ConanPackages')
 
           package.destroy
         end
@@ -341,11 +345,5 @@ module API
         end
       end
     end
-
-    helpers do
-      include Gitlab::Utils::StrongMemoize
-      include ::API::Helpers::RelatedResourcesHelpers
-      include ::API::Helpers::Packages::Conan::ApiHelpers
-    end
   end
 end
diff --git a/lib/api/conan_project_packages.rb b/lib/api/conan_project_packages.rb
new file mode 100644
index 00000000000..c51992231a7
--- /dev/null
+++ b/lib/api/conan_project_packages.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# Conan Project-Level Package Manager Client API
+module API
+  class ConanProjectPackages < Grape::API::Instance
+    params do
+      requires :id, type: Integer, desc: 'The ID of a project', regexp: %r{\A[1-9]\d*\z}
+    end
+
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      namespace ':id/packages/conan/v1' do
+        include ConanPackageEndpoints
+      end
+    end
+  end
+end
diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb
index 5b513ce1aab..dcbf933a4e1 100644
--- a/lib/api/helpers/packages/conan/api_helpers.rb
+++ b/lib/api/helpers/packages/conan/api_helpers.rb
@@ -5,6 +5,8 @@ module API
     module Packages
       module Conan
         module ApiHelpers
+          include Gitlab::Utils::StrongMemoize
+
           def present_download_urls(entity)
             authorize!(:read_package, project)
 
@@ -31,7 +33,7 @@ module API
           def recipe_upload_urls
             { upload_urls: Hash[
               file_names.select(&method(:recipe_file?)).map do |file_name|
-                [file_name, recipe_file_upload_url(file_name)]
+                [file_name, build_recipe_file_upload_url(file_name)]
               end
             ] }
           end
@@ -39,7 +41,7 @@ module API
           def package_upload_urls
             { upload_urls: Hash[
               file_names.select(&method(:package_file?)).map do |file_name|
-                [file_name, package_file_upload_url(file_name)]
+                [file_name, build_package_file_upload_url(file_name)]
               end
             ] }
           end
@@ -52,32 +54,58 @@ module API
             file_name.in?(::Packages::Conan::FileMetadatum::PACKAGE_FILES)
           end
 
-          def package_file_upload_url(file_name)
-            expose_url(
-              api_v4_packages_conan_v1_files_package_path(
-                package_name: params[:package_name],
-                package_version: params[:package_version],
-                package_username: params[:package_username],
-                package_channel: params[:package_channel],
-                recipe_revision: '0',
-                conan_package_reference: params[:conan_package_reference],
-                package_revision: '0',
-                file_name: file_name
-              )
+          def build_package_file_upload_url(file_name)
+            options = url_options(file_name).merge(
+              conan_package_reference: params[:conan_package_reference],
+              package_revision: ::Packages::Conan::FileMetadatum::DEFAULT_PACKAGE_REVISION
             )
+
+            package_file_url(options)
+          end
+
+          def build_recipe_file_upload_url(file_name)
+            recipe_file_url(url_options(file_name))
           end
 
-          def recipe_file_upload_url(file_name)
-            expose_url(
-              api_v4_packages_conan_v1_files_export_path(
-                package_name: params[:package_name],
-                package_version: params[:package_version],
-                package_username: params[:package_username],
-                package_channel: params[:package_channel],
-                recipe_revision: '0',
-                file_name: file_name
+          def url_options(file_name)
+            {
+              package_name: params[:package_name],
+              package_version: params[:package_version],
+              package_username: params[:package_username],
+              package_channel: params[:package_channel],
+              file_name: file_name,
+              recipe_revision: ::Packages::Conan::FileMetadatum::DEFAULT_RECIPE_REVISION
+            }
+          end
+
+          def package_file_url(options)
+            case package_scope
+            when :project
+              expose_url(
+                api_v4_projects_packages_conan_v1_files_package_path(
+                  options.merge(id: project.id)
+                )
               )
-            )
+            when :instance
+              expose_url(
+                api_v4_packages_conan_v1_files_package_path(options)
+              )
+            end
+          end
+
+          def recipe_file_url(options)
+            case package_scope
+            when :project
+              expose_url(
+                api_v4_projects_packages_conan_v1_files_export_path(
+                  options.merge(id: project.id)
+                )
+              )
+            when :instance
+              expose_url(
+                api_v4_packages_conan_v1_files_export_path(options)
+              )
+            end
           end
 
           def recipe
@@ -86,8 +114,13 @@ module API
 
           def project
             strong_memoize(:project) do
-              full_path = ::Packages::Conan::Metadatum.full_path_from(package_username: params[:package_username])
-              Project.find_by_full_path(full_path)
+              case package_scope
+              when :project
+                find_project!(params[:id])
+              when :instance
+                full_path = ::Packages::Conan::Metadatum.full_path_from(package_username: params[:package_username])
+                find_project!(full_path)
+              end
             end
           end
 
@@ -97,6 +130,7 @@ module API
                 .conan
                 .with_name(params[:package_name])
                 .with_version(params[:package_version])
+                .with_conan_username(params[:package_username])
                 .with_conan_channel(params[:package_channel])
                 .order_created
                 .last
@@ -124,7 +158,7 @@ module API
                 conan_package_reference: params[:conan_package_reference]
               ).execute!
 
-            package_event('pull_package') if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY
+            package_event('pull_package', category: 'API::ConanPackages') if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY
 
             present_carrierwave_file!(package_file.file)
           end
@@ -135,7 +169,7 @@ module API
 
           def track_push_package_event
             if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY && params[:file].size > 0 # rubocop: disable Style/ZeroLengthPredicate
-              package_event('push_package')
+              package_event('push_package', category: 'API::ConanPackages')
             end
           end
 
@@ -236,6 +270,10 @@ module API
 
             token
           end
+
+          def package_scope
+            params[:id].present? ? :project : :instance
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
index a078f203cf2..c3a92b67a8b 100644
--- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
@@ -118,6 +118,9 @@ apifuzzer_fuzz:
         # Wait for testing to complete if api fuzzer is scanning
         - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI" != "" ]; then echo "Waiting for API Fuzzer to exit"; docker wait apifuzzer; fi
         #
+        # Propagate exit code from api fuzzer (if any)
+        - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing exited with an error. Logs are available as job artifacts."; docker logs apifuzzer; exit 1; fi
+        #
         # Run user provided pre-script
         - sh -c "$FUZZAPI_POST_SCRIPT"
         #
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 3783a340257..8ace4157ad7 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -19,15 +19,15 @@ module Gitlab
       GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout
       EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'
 
-      NoRepository = Class.new(StandardError)
-      InvalidRepository = Class.new(StandardError)
-      InvalidBlobName = Class.new(StandardError)
-      InvalidRef = Class.new(StandardError)
-      GitError = Class.new(StandardError)
-      DeleteBranchError = Class.new(StandardError)
-      TagExistsError = Class.new(StandardError)
-      ChecksumError = Class.new(StandardError)
-      class CreateTreeError < StandardError
+      NoRepository = Class.new(::Gitlab::Git::BaseError)
+      InvalidRepository = Class.new(::Gitlab::Git::BaseError)
+      InvalidBlobName = Class.new(::Gitlab::Git::BaseError)
+      InvalidRef = Class.new(::Gitlab::Git::BaseError)
+      GitError = Class.new(::Gitlab::Git::BaseError)
+      DeleteBranchError = Class.new(::Gitlab::Git::BaseError)
+      TagExistsError = Class.new(::Gitlab::Git::BaseError)
+      ChecksumError = Class.new(::Gitlab::Git::BaseError)
+      class CreateTreeError < ::Gitlab::Git::BaseError
         attr_reader :error_code
 
         def initialize(error_code)
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb
index bb0c18735bb..6417ec20960 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb
@@ -5,7 +5,7 @@ module Gitlab
     module DuplicateJobs
       class Client
         def call(worker_class, job, queue, _redis_pool, &block)
-          DuplicateJob.new(job, queue).schedule(&block)
+          ::Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(job, queue).schedule(&block)
         end
       end
     end
diff --git a/lib/gitlab/static_site_editor/config/combined_config.rb b/lib/gitlab/static_site_editor/config/combined_config.rb
deleted file mode 100644
index 387ce08e552..00000000000
--- a/lib/gitlab/static_site_editor/config/combined_config.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module StaticSiteEditor
-    module Config
-      class CombinedConfig
-        def initialize(repository, ref, path, return_url)
-          @repository = repository
-          @ref = ref
-          @path = path
-          @return_url = return_url
-        end
-
-        def data
-          generated_data = Gitlab::StaticSiteEditor::Config::GeneratedConfig.new(
-            @repository,
-            @ref,
-            @path,
-            @return_url
-          ).data
-          file_data = Gitlab::StaticSiteEditor::Config::FileConfig.new.data
-          check_for_duplicate_keys(generated_data, file_data)
-          generated_data.merge(file_data)
-        end
-
-        private
-
-        def check_for_duplicate_keys(generated_data, file_data)
-          duplicate_keys = generated_data.keys & file_data.keys
-          raise StandardError.new("Duplicate key(s) '#{duplicate_keys}' found.") if duplicate_keys.present?
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/static_site_editor/config/file_config.rb b/lib/gitlab/static_site_editor/config/file_config.rb
index b43a5440e6b..f647c85e1c8 100644
--- a/lib/gitlab/static_site_editor/config/file_config.rb
+++ b/lib/gitlab/static_site_editor/config/file_config.rb
@@ -5,9 +5,8 @@ module Gitlab
     module Config
       class FileConfig
         def data
-          merge_requests_illustration_path = ActionController::Base.helpers.image_path('illustrations/merge_requests.svg')
           {
-            merge_requests_illustration_path: merge_requests_illustration_path
+            static_site_generator: 'middleman'
           }
         end
       end
diff --git a/lib/gitlab/static_site_editor/config/generated_config.rb b/lib/gitlab/static_site_editor/config/generated_config.rb
index 4c875b1d211..f3dce74a32f 100644
--- a/lib/gitlab/static_site_editor/config/generated_config.rb
+++ b/lib/gitlab/static_site_editor/config/generated_config.rb
@@ -14,6 +14,7 @@ module Gitlab
         end
 
         def data
+          merge_requests_illustration_path = ActionController::Base.helpers.image_path('illustrations/merge_requests.svg')
           {
             branch: ref,
             path: path,
@@ -23,7 +24,8 @@ module Gitlab
             namespace: project.namespace.full_path,
             return_url: sanitize_url(return_url),
             is_supported_content: supported_content?.to_s,
-            base_url: Gitlab::Routing.url_helpers.project_show_sse_path(project, full_path)
+            base_url: Gitlab::Routing.url_helpers.project_show_sse_path(project, full_path),
+            merge_requests_illustration_path: merge_requests_illustration_path
           }
         end
 
diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb
index f38050ceee6..55940f51f99 100644
--- a/lib/gitlab/usage_data_queries.rb
+++ b/lib/gitlab/usage_data_queries.rb
@@ -11,6 +11,14 @@ module Gitlab
         raw_sql(relation, column, :distinct)
       end
 
+      def redis_usage_data(counter = nil, &block)
+        if block_given?
+          { redis_usage_data_block: block.to_s }
+        elsif counter.present?
+          { redis_usage_data_counter: counter }
+        end
+      end
+
       private
 
       def raw_sql(relation, column, distinct = nil)