diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-10-20 09:40:42 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-10-20 09:40:42 +0000 |
| commit | ee664acb356f8123f4f6b00b73c1e1cf0866c7fb (patch) | |
| tree | f8479f94a28f66654c6a4f6fb99bad6b4e86a40e /lib | |
| parent | 62f7d5c5b69180e82ae8196b7b429eeffc8e7b4f (diff) | |
| download | gitlab-ce-15.5.0-rc42.tar.gz | |
Add latest changes from gitlab-org/gitlab@15-5-stable-eev15.5.0-rc42
Diffstat (limited to 'lib')
414 files changed, 18140 insertions, 3521 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index e6ce62a1c6e..74f6515f07f 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -12,7 +12,8 @@ module API %w[group project].each do |source_type| params do - requires :id, type: String, desc: "The #{source_type} ID" + requires :id, type: String, + desc: "The ID or URL-encoded path of the #{source_type} owned by the authenticated user" end resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Gets a list of access requests for a #{source_type}." do @@ -54,7 +55,8 @@ module API end params do requires :user_id, type: Integer, desc: 'The user ID of the access requester' - optional :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' + optional :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, the Developer role)', + default: 30 end # rubocop: disable CodeReuse/ActiveRecord put ':id/access_requests/:user_id/approve' do diff --git a/lib/api/admin/batched_background_migrations.rb b/lib/api/admin/batched_background_migrations.rb index 675f3365bd3..e8cc08a23be 100644 --- a/lib/api/admin/batched_background_migrations.rb +++ b/lib/api/admin/batched_background_migrations.rb @@ -61,6 +61,11 @@ module API end put do Gitlab::Database::SharedModel.using_connection(base_model.connection) do + unless batched_background_migration.paused? + msg = 'You can resume only `paused` batched background migrations.' + render_api_error!(msg, 422) + end + batched_background_migration.execute! present_entity(batched_background_migration) end @@ -81,6 +86,11 @@ module API end put do Gitlab::Database::SharedModel.using_connection(base_model.connection) do + unless batched_background_migration.active? + msg = 'You can pause only `active` batched background migrations.' + render_api_error!(msg, 422) + end + batched_background_migration.pause! present_entity(batched_background_migration) end diff --git a/lib/api/alert_management_alerts.rb b/lib/api/alert_management_alerts.rb index bbb7e7280c9..f03f133f6f7 100644 --- a/lib/api/alert_management_alerts.rb +++ b/lib/api/alert_management_alerts.rb @@ -32,7 +32,8 @@ module API success Entities::MetricImage end params do - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The image file to be uploaded' + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The image file to be uploaded', + documentation: { type: 'file' } optional :url, type: String, desc: 'The url to view more metric info' optional :url_text, type: String, desc: 'A description of the image or URL' end diff --git a/lib/api/api.rb b/lib/api/api.rb index 443bf1d649a..933c3f69075 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -3,6 +3,7 @@ module API class API < ::API::Base include APIGuard + include Helpers::OpenApi LOG_FILENAME = Rails.root.join("log", "api_json.log") @@ -165,6 +166,13 @@ module API ::Users::ActivityService.new(@current_user).execute end + # Mount endpoints to include in the OpenAPI V2 documentation here + namespace do + mount ::API::Metadata + + add_open_api_documentation! + end + # Keep in alphabetical order mount ::API::AccessRequests mount ::API::Admin::BatchedBackgroundMigrations @@ -250,7 +258,6 @@ module API mount ::API::MergeRequestApprovals mount ::API::MergeRequestDiffs mount ::API::MergeRequests - mount ::API::Metadata mount ::API::Metrics::Dashboard::Annotations mount ::API::Metrics::UserStarredDashboards mount ::API::Namespaces @@ -263,7 +270,7 @@ module API mount ::API::PackageFiles mount ::API::Pages mount ::API::PagesDomains - mount ::API::PersonalAccessTokens::SelfRevocation + mount ::API::PersonalAccessTokens::SelfInformation mount ::API::PersonalAccessTokens mount ::API::ProjectClusters mount ::API::ProjectContainerRepositories @@ -316,7 +323,6 @@ module API mount ::API::UsageDataQueries mount ::API::UserCounts mount ::API::Users - mount ::API::Version mount ::API::Wikis mount ::API::Ml::Mlflow end diff --git a/lib/api/applications.rb b/lib/api/applications.rb index 70488621f33..4048215160f 100644 --- a/lib/api/applications.rb +++ b/lib/api/applications.rb @@ -41,6 +41,9 @@ module API end desc 'Delete an application' + params do + requires :id, type: Integer, desc: 'The ID of the application (not the application_id)' + end delete ':id' do application = ApplicationsFinder.new(params).execute break not_found!('Application') unless application diff --git a/lib/api/badges.rb b/lib/api/badges.rb index f969eec8431..0a3f247ffd6 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -31,6 +31,7 @@ module API end params do use :pagination + optional :name, type: String, desc: 'Name for the badge' end get ":id/badges", urgency: :low do source = find_source(source_type, params[:id]) diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 5588818cbaf..7e6b0214c03 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -52,19 +52,13 @@ module API merged_branch_names = repository.merged_branch_names(branches.map(&:name)) - expiry_time = if Feature.enabled?(:increase_branch_cache_expiry, type: :ops) - 60.minutes - else - 10.minutes - end - present_cached( branches, with: Entities::Branch, current_user: current_user, project: user_project, merged_branch_names: merged_branch_names, - expires_in: expiry_time, + expires_in: 60.minutes, cache_context: -> (branch) { [current_user&.cache_key, merged_branch_names.include?(branch.name)] } ) end diff --git a/lib/api/bulk_imports.rb b/lib/api/bulk_imports.rb index 2c6adc0f37b..c54632919be 100644 --- a/lib/api/bulk_imports.rb +++ b/lib/api/bulk_imports.rb @@ -32,11 +32,16 @@ module API end end - before { authenticate! } + before do + not_found! unless ::BulkImports::Features.enabled? + + authenticate! + end resource :bulk_imports do desc 'Start a new GitLab Migration' do detail 'This feature was introduced in GitLab 14.2.' + success Entities::BulkImport end params do requires :configuration, type: Hash, desc: 'The source GitLab instance configuration' do @@ -83,6 +88,7 @@ module API desc 'List all GitLab Migrations' do detail 'This feature was introduced in GitLab 14.1.' + success Entities::BulkImport end params do use :pagination @@ -97,6 +103,7 @@ module API desc "List all GitLab Migrations' entities" do detail 'This feature was introduced in GitLab 14.1.' + success Entities::BulkImports::Entity end params do use :pagination @@ -116,6 +123,7 @@ module API desc 'Get GitLab Migration details' do detail 'This feature was introduced in GitLab 14.1.' + success Entities::BulkImport end params do requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration" @@ -126,6 +134,7 @@ module API desc "List GitLab Migration entities" do detail 'This feature was introduced in GitLab 14.1.' + success Entities::BulkImports::Entity end params do requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration" @@ -139,6 +148,7 @@ module API desc 'Get GitLab Migration entity details' do detail 'This feature was introduced in GitLab 14.1.' + success Entities::BulkImports::Entity end params do requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration" diff --git a/lib/api/ci/job_artifacts.rb b/lib/api/ci/job_artifacts.rb index b3a0a9ef54a..37c7cc73c46 100644 --- a/lib/api/ci/job_artifacts.rb +++ b/lib/api/ci/job_artifacts.rb @@ -38,7 +38,7 @@ module API latest_build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) authorize_read_job_artifacts!(latest_build) - present_artifacts_file!(latest_build.artifacts_file) + present_artifacts_file!(latest_build.artifacts_file, project: latest_build.project) end desc 'Download a specific file from artifacts archive from a ref' do @@ -80,7 +80,7 @@ module API build = find_build!(params[:job_id]) authorize_read_job_artifacts!(build) - present_artifacts_file!(build.artifacts_file) + present_artifacts_file!(build.artifacts_file, project: build.project) end desc 'Download a specific file from artifacts archive' do diff --git a/lib/api/ci/resource_groups.rb b/lib/api/ci/resource_groups.rb index e3fd887475a..ea6d3cc8fd4 100644 --- a/lib/api/ci/resource_groups.rb +++ b/lib/api/ci/resource_groups.rb @@ -38,6 +38,25 @@ module API present resource_group, with: Entities::Ci::ResourceGroup end + desc 'List upcoming jobs of a resource group' do + success Entities::Ci::JobBasic + end + params do + requires :key, type: String, desc: 'The key of the resource group' + + use :pagination + end + get ':id/resource_groups/:key/upcoming_jobs' do + authorize! :read_resource_group, resource_group + authorize! :read_build, user_project + + upcoming_processables = resource_group + .upcoming_processables + .preload(:user, pipeline: :project) # rubocop:disable CodeReuse/ActiveRecord + + present paginate(upcoming_processables), with: Entities::Ci::JobBasic + end + desc 'Edit a resource group' do success Entities::Ci::ResourceGroup end diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 9e4a700d0f3..2d2dcc544f9 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -332,7 +332,7 @@ module API authenticate_job!(require_running: false) end - present_artifacts_file!(current_job.artifacts_file, supports_direct_download: params[:direct_download]) + present_artifacts_file!(current_job.artifacts_file, project: current_job.project, supports_direct_download: params[:direct_download]) end end end diff --git a/lib/api/ci/secure_files.rb b/lib/api/ci/secure_files.rb index c1f47dd67ce..68431df203b 100644 --- a/lib/api/ci/secure_files.rb +++ b/lib/api/ci/secure_files.rb @@ -74,6 +74,10 @@ module API file_too_large! unless secure_file.file.size < ::Ci::SecureFile::FILE_SIZE_LIMIT.to_i if secure_file.save + if Feature.enabled?(:secure_files_metadata_parsers, user_project) + ::Ci::ParseSecureFileMetadataWorker.perform_async(secure_file.id) # rubocop:disable CodeReuse/Worker + end + present secure_file, with: Entities::Ci::SecureFile else render_validation_error!(secure_file) diff --git a/lib/api/ci/variables.rb b/lib/api/ci/variables.rb index f9707960b9d..c9e1d115d03 100644 --- a/lib/api/ci/variables.rb +++ b/lib/api/ci/variables.rb @@ -33,6 +33,9 @@ module API end params do requires :key, type: String, desc: 'The key of the variable' + optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production' do + optional :environment_scope, type: String, desc: 'The environment scope of the variable' + end end # rubocop: disable CodeReuse/ActiveRecord get ':id/variables/:key', urgency: :low do @@ -78,7 +81,9 @@ module API optional :masked, type: Boolean, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: ::Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' optional :environment_scope, type: String, desc: 'The environment_scope of the variable' - optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production' + optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production' do + optional :environment_scope, type: String, desc: 'The environment scope of the variable' + end end # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do @@ -104,7 +109,9 @@ module API end params do requires :key, type: String, desc: 'The key of the variable' - optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production' + optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production' do + optional :environment_scope, type: String, desc: 'The environment scope of the variable' + end end # rubocop: disable CodeReuse/ActiveRecord delete ':id/variables/:key' do diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 5a6d06dcdd9..7d8b58fd7b6 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -52,8 +52,8 @@ module API optional :ref, type: String, desc: 'The ref' optional :target_url, type: String, desc: 'The target URL to associate with this status' optional :description, type: String, desc: 'A short description of the status' - optional :name, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' - optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' + optional :name, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"', documentation: { default: 'default' } + optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"', documentation: { default: 'default' } optional :coverage, type: Float, desc: 'The total code coverage' optional :pipeline_id, type: Integer, desc: 'An existing pipeline ID, when multiple pipelines on the same commit SHA have been triggered' end diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb index d6e006df976..4cc680068b6 100644 --- a/lib/api/concerns/packages/npm_endpoints.rb +++ b/lib/api/concerns/packages/npm_endpoints.rb @@ -116,7 +116,12 @@ module API redirect_request = project_or_nil.blank? || packages.empty? - redirect_registry_request(redirect_request, :npm, package_name: package_name) do + redirect_registry_request( + forward_to_registry: redirect_request, + package_type: :npm, + target: project_or_nil, + package_name: package_name + ) do authorize_read_package!(project) not_found!('Packages') if packages.empty? diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index 9dedc4390f7..03f0f97b805 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -81,11 +81,7 @@ module API package = ::Packages::Debian::FindOrCreateIncomingService.new(authorized_user_project, current_user).execute - package_file = ::Packages::Debian::CreatePackageFileService.new(package, file_params).execute - - if params['file_name'].end_with? '.changes' - ::Packages::Debian::ProcessChangesWorker.perform_async(package_file.id, current_user.id) # rubocop:disable CodeReuse/Worker - end + ::Packages::Debian::CreatePackageFileService.new(package: package, current_user: current_user, params: file_params).execute created! rescue ObjectStorage::RemoteStoreError => e diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index ca13db8701e..c53f4bca5a7 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -161,9 +161,7 @@ module API end end - desc 'Delete deploy key for a project' do - success Key - end + desc 'Delete deploy key for a project' params do requires :key_id, type: Integer, desc: 'The ID of the deploy key' end diff --git a/lib/api/entities/bulk_imports/entity_failure.rb b/lib/api/entities/bulk_imports/entity_failure.rb index a3dbe3280ee..56312745868 100644 --- a/lib/api/entities/bulk_imports/entity_failure.rb +++ b/lib/api/entities/bulk_imports/entity_failure.rb @@ -4,11 +4,16 @@ module API module Entities module BulkImports class EntityFailure < Grape::Entity - expose :pipeline_class - expose :pipeline_step + expose :relation + expose :pipeline_step, as: :step + expose :exception_message do |failure| + ::Projects::ImportErrorFilter.filter_message(failure.exception_message.truncate(72)) + end expose :exception_class expose :correlation_id_value expose :created_at + expose :pipeline_class + expose :pipeline_step end end end diff --git a/lib/api/entities/ci/job_basic.rb b/lib/api/entities/ci/job_basic.rb index 3d9318ec428..fb975475cf5 100644 --- a/lib/api/entities/ci/job_basic.rb +++ b/lib/api/entities/ci/job_basic.rb @@ -21,7 +21,7 @@ module API expose :project do expose :ci_job_token_scope_enabled do |job| - job.project.ci_job_token_scope_enabled? + job.project.ci_outbound_job_token_scope_enabled? end end end diff --git a/lib/api/entities/ci/runner.rb b/lib/api/entities/ci/runner.rb index e29d55771f2..f034eb5c94c 100644 --- a/lib/api/entities/ci/runner.rb +++ b/lib/api/entities/ci/runner.rb @@ -7,7 +7,7 @@ module API expose :id expose :description expose :ip_address - expose :active # TODO Remove in %16.0 in favor of `paused` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/351109 + expose :active # TODO Remove in v5 in favor of `paused` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/375709 expose :paused do |runner| !runner.active end @@ -16,7 +16,7 @@ module API expose :name expose :online?, as: :online # DEPRECATED - # TODO Remove in %16.0 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 + # TODO Remove in v5 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/375709 expose :deprecated_rest_status, as: :status end end diff --git a/lib/api/entities/license.rb b/lib/api/entities/license.rb index d7a414344c1..8ecf8a430fe 100644 --- a/lib/api/entities/license.rb +++ b/lib/api/entities/license.rb @@ -2,6 +2,7 @@ module API module Entities + # Serializes a Licensee::License class License < Entities::LicenseBasic expose :popular?, as: :popular expose(:description) { |license| license.meta['description'] } diff --git a/lib/api/entities/license_basic.rb b/lib/api/entities/license_basic.rb index 08af68785a9..0916738d21d 100644 --- a/lib/api/entities/license_basic.rb +++ b/lib/api/entities/license_basic.rb @@ -2,10 +2,16 @@ module API module Entities + # Serializes a Gitlab::Git::DeclaredLicense class LicenseBasic < Grape::Entity expose :key, :name, :nickname expose :url, as: :html_url - expose(:source_url) { |license| license.meta['source'] } + + # This was dropped: + # https://github.com/github/choosealicense.com/commit/325806b42aa3d5b78e84120327ec877bc936dbdd#diff-66df8f1997786f7052d29010f2cbb4c66391d60d24ca624c356acc0ab986f139 + expose :source_url do |_| + nil + end end end end diff --git a/lib/api/entities/merge_request_approvals.rb b/lib/api/entities/merge_request_approvals.rb index 0c464844ae7..6810952b2fc 100644 --- a/lib/api/entities/merge_request_approvals.rb +++ b/lib/api/entities/merge_request_approvals.rb @@ -8,7 +8,7 @@ module API end expose :user_can_approve do |merge_request, options| - merge_request.can_be_approved_by?(options[:current_user]) + merge_request.eligible_for_approval_by?(options[:current_user]) end expose :approved do |merge_request| diff --git a/lib/api/entities/metadata.rb b/lib/api/entities/metadata.rb new file mode 100644 index 00000000000..daa491ec42a --- /dev/null +++ b/lib/api/entities/metadata.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module API + module Entities + class Metadata < Grape::Entity + expose :version + expose :revision + expose :kas do + expose :enabled, documentation: { type: 'boolean' } + expose :externalUrl + expose :version + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/experiment.rb b/lib/api/entities/ml/mlflow/experiment.rb index cfe366feaab..54e0fe63985 100644 --- a/lib/api/entities/ml/mlflow/experiment.rb +++ b/lib/api/entities/ml/mlflow/experiment.rb @@ -5,22 +5,10 @@ module API module Ml module Mlflow class Experiment < Grape::Entity - expose :experiment do - expose :experiment_id - expose :name - expose :lifecycle_stage - expose :artifact_location - end - - private - - def lifecycle_stage - object.deleted_on? ? 'deleted' : 'active' - end - - def experiment_id - object.iid.to_s - end + expose(:experiment_id) { |experiment| experiment.iid.to_s } + expose :name + expose(:lifecycle_stage) { |experiment| experiment.deleted_on? ? 'deleted' : 'active' } + expose(:artifact_location) { |experiment| 'not_implemented' } end end end diff --git a/lib/api/entities/ml/mlflow/get_experiment.rb b/lib/api/entities/ml/mlflow/get_experiment.rb new file mode 100644 index 00000000000..f28d2ce76f6 --- /dev/null +++ b/lib/api/entities/ml/mlflow/get_experiment.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class GetExperiment < Grape::Entity + expose :itself, using: Experiment, as: :experiment + end + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/list_experiment.rb b/lib/api/entities/ml/mlflow/list_experiment.rb new file mode 100644 index 00000000000..515015bf4b7 --- /dev/null +++ b/lib/api/entities/ml/mlflow/list_experiment.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class ListExperiment < Grape::Entity + expose :experiments, with: Experiment + end + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/metric.rb b/lib/api/entities/ml/mlflow/metric.rb new file mode 100644 index 00000000000..963aaa5f144 --- /dev/null +++ b/lib/api/entities/ml/mlflow/metric.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class Metric < Grape::Entity + expose :name, as: :key + expose :value + expose :tracked_at, as: :timestamp + expose :step, expose_nil: false + end + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/run.rb b/lib/api/entities/ml/mlflow/run.rb index c679330206e..a8e1cfe08dd 100644 --- a/lib/api/entities/ml/mlflow/run.rb +++ b/lib/api/entities/ml/mlflow/run.rb @@ -7,7 +7,10 @@ module API class Run < Grape::Entity expose :run do expose(:info) { |candidate| RunInfo.represent(candidate) } - expose(:data) { |candidate| {} } + expose :data do + expose :metrics, using: Metric + expose :params, using: RunParam + end end end end diff --git a/lib/api/entities/ml/mlflow/run_param.rb b/lib/api/entities/ml/mlflow/run_param.rb new file mode 100644 index 00000000000..75fee738f8b --- /dev/null +++ b/lib/api/entities/ml/mlflow/run_param.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class RunParam < Grape::Entity + expose :name, as: :key + expose :value + end + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/update_run.rb b/lib/api/entities/ml/mlflow/update_run.rb index 5acdaab0e33..090d69b8895 100644 --- a/lib/api/entities/ml/mlflow/update_run.rb +++ b/lib/api/entities/ml/mlflow/update_run.rb @@ -10,7 +10,7 @@ module API private def run_info - ::API::Entities::Ml::Mlflow::RunInfo.represent object + RunInfo.represent object end end end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 1739bdd639e..f158695f605 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -80,6 +80,7 @@ module API expose(:analytics_access_level) { |project, options| project_feature_string_access_level(project, :analytics) } expose(:container_registry_access_level) { |project, options| project_feature_string_access_level(project, :container_registry) } expose(:security_and_compliance_access_level) { |project, options| project_feature_string_access_level(project, :security_and_compliance) } + expose(:releases_access_level) { |project, options| project_feature_string_access_level(project, :releases) } expose :emails_disabled expose :shared_runners_enabled @@ -103,7 +104,7 @@ module API expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :ci_default_git_depth expose :ci_forward_deployment_enabled - expose :ci_job_token_scope_enabled + expose(:ci_job_token_scope_enabled) { |p, _| p.ci_outbound_job_token_scope_enabled? } expose :ci_separated_caches expose :ci_opt_in_jwt expose :ci_allow_fork_pipelines_to_run_in_parent_project diff --git a/lib/api/entities/user_with_admin.rb b/lib/api/entities/user_with_admin.rb index f9c1a646a4f..53fef7a46e2 100644 --- a/lib/api/entities/user_with_admin.rb +++ b/lib/api/entities/user_with_admin.rb @@ -6,6 +6,7 @@ module API expose :admin?, as: :is_admin expose :note expose :namespace_id + expose :created_by, with: UserBasic end end end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index c4b67f83941..42d5e6a73b3 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -40,7 +40,7 @@ module API params do requires :name, type: String, desc: 'The name of the environment to be created' optional :external_url, type: String, desc: 'URL on which this deployment is viewable' - optional :slug, absence: { message: "is automatically generated and cannot be changed" } + optional :slug, absence: { message: "is automatically generated and cannot be changed" }, documentation: { hidden: true } optional :tier, type: String, values: Environment.tiers.keys, desc: 'The tier of the environment to be created' end post ':id/environments' do @@ -64,7 +64,7 @@ module API # TODO: disallow renaming via the API https://gitlab.com/gitlab-org/gitlab/-/issues/338897 optional :name, type: String, desc: 'DEPRECATED: Renaming environment can lead to errors, this will be removed in 15.0' optional :external_url, type: String, desc: 'The new URL on which this deployment is viewable' - optional :slug, absence: { message: "is automatically generated and cannot be changed" } + optional :slug, absence: { message: "is automatically generated and cannot be changed" }, documentation: { hidden: true } optional :tier, type: String, values: Environment.tiers.keys, desc: 'The tier of the environment to be created' end put ':id/environments/:environment_id' do diff --git a/lib/api/features.rb b/lib/api/features.rb index f89da48acea..9d4e6eee82c 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -7,6 +7,7 @@ module API feature_category :feature_flags urgency :low + # TODO: remove these helpers with feature flag set_feature_flag_service helpers do def gate_value(params) case params[:value] @@ -87,35 +88,49 @@ module API mutually_exclusive :key, :project end post ':name' do - validate_feature_flag_name!(params[:name]) unless params[:force] - - targets = gate_targets(params) - value = gate_value(params) - key = gate_key(params) - - case value - when true - if gate_specified?(params) - targets.each { |target| Feature.enable(params[:name], target) } - else - Feature.enable(params[:name]) - end - when false - if gate_specified?(params) - targets.each { |target| Feature.disable(params[:name], target) } + if Feature.enabled?(:set_feature_flag_service) + flag_params = declared_params(include_missing: false) + response = ::Admin::SetFeatureFlagService + .new(feature_flag_name: params[:name], params: flag_params) + .execute + + if response.success? + present response.payload[:feature_flag], + with: Entities::Feature, current_user: current_user else - Feature.disable(params[:name]) + bad_request!(response.message) end else - if key == :percentage_of_actors - Feature.enable_percentage_of_actors(params[:name], value) + validate_feature_flag_name!(params[:name]) unless params[:force] + + targets = gate_targets(params) + value = gate_value(params) + key = gate_key(params) + + case value + when true + if gate_specified?(params) + targets.each { |target| Feature.enable(params[:name], target) } + else + Feature.enable(params[:name]) + end + when false + if gate_specified?(params) + targets.each { |target| Feature.disable(params[:name], target) } + else + Feature.disable(params[:name]) + end else - Feature.enable_percentage_of_time(params[:name], value) + if key == :percentage_of_actors + Feature.enable_percentage_of_actors(params[:name], value) + else + Feature.enable_percentage_of_time(params[:name], value) + end end - end - present Feature.get(params[:name]), # rubocop:disable Gitlab/AvoidFeatureGet - with: Entities::Feature, current_user: current_user + present Feature.get(params[:name]), # rubocop:disable Gitlab/AvoidFeatureGet + with: Entities::Feature, current_user: current_user + end rescue Feature::Target::UnknowTargetError => e bad_request!(e.message) end @@ -128,6 +143,7 @@ module API end end + # TODO: remove this helper with feature flag set_feature_flag_service helpers do def validate_feature_flag_name!(name) # no-op diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index ad5455c5de6..0098b074f05 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -40,6 +40,8 @@ module API end put 'authorize' do + project = authorized_user_project + authorize_workhorse!(subject: project, maximum_size: project.actual_limits.generic_packages_max_file_size) end @@ -59,6 +61,8 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true put do + project = authorized_user_project + authorize_upload!(project) bad_request!('File is too large') if max_file_size_exceeded? @@ -95,6 +99,8 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true get do + project = authorized_user_project(action: :read_package) + authorize_read_package!(project) package = ::Packages::Generic::PackageFinder.new(project).execute!(params[:package_name], params[:package_version]) @@ -112,12 +118,8 @@ module API include ::API::Helpers::PackagesHelpers include ::API::Helpers::Packages::BasicAuthHelpers - def project - authorized_user_project - end - def max_file_size_exceeded? - project.actual_limits.exceeded?(:generic_packages_max_file_size, params[:file].size) + authorized_user_project.actual_limits.exceeded?(:generic_packages_max_file_size, params[:file].size) end end end diff --git a/lib/api/group_import.rb b/lib/api/group_import.rb index abb8c10efc6..cef9b542c9e 100644 --- a/lib/api/group_import.rb +++ b/lib/api/group_import.rb @@ -54,7 +54,7 @@ module API params do requires :path, type: String, desc: 'Group path' requires :name, type: String, desc: 'Group name' - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The group export file to be imported' + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The group export file to be imported', documentation: { type: 'file' } optional :parent_id, type: Integer, desc: "The ID of the parent group that the group will be imported into. Defaults to the current user's namespace." end post 'import' do diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 6b1fc0d4279..ca99e30fbf7 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -123,6 +123,12 @@ module API end def present_groups_with_pagination_strategies(params, groups) + # Prevent Rails from optimizing the count query and inadvertadly creating a poor performing databse query. + # https://gitlab.com/gitlab-org/gitlab/-/issues/368969 + if Feature.enabled?(:present_groups_select_all) + groups = groups.select(groups.arel_table[Arel.star]) + end + return present_groups(params, groups) if current_user.present? options = { diff --git a/lib/api/helm_packages.rb b/lib/api/helm_packages.rb index f90084a7e57..fa2537bcfc4 100644 --- a/lib/api/helm_packages.rb +++ b/lib/api/helm_packages.rb @@ -44,9 +44,10 @@ module API end get ":channel/index.yaml" do - authorize_read_package!(authorized_user_project) + project = authorized_user_project(action: :read_package) + authorize_read_package!(project) - packages = Packages::Helm::PackagesFinder.new(authorized_user_project, params[:channel]).execute + packages = Packages::Helm::PackagesFinder.new(project, params[:channel]).execute env['api.format'] = :yaml present ::Packages::Helm::IndexPresenter.new(params[:id], params[:channel], packages), @@ -61,11 +62,12 @@ module API requires :file_name, type: String, desc: 'Helm package file name' end get ":channel/charts/:file_name.tgz" do - authorize_read_package!(authorized_user_project) + project = authorized_user_project(action: :read_package) + authorize_read_package!(project) - package_file = Packages::Helm::PackageFilesFinder.new(authorized_user_project, params[:channel], file_name: "#{params[:file_name]}.tgz").most_recent! + package_file = Packages::Helm::PackageFilesFinder.new(project, params[:channel], file_name: "#{params[:file_name]}.tgz").most_recent! - track_package_event('pull_package', :helm, project: authorized_user_project, namespace: authorized_user_project.namespace) + track_package_event('pull_package', :helm, project: project, namespace: project.namespace) present_package_file!(package_file) end @@ -89,7 +91,7 @@ module API end params do requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex - requires :chart, type: ::API::Validations::Types::WorkhorseFile, desc: 'The chart file to be published (generated by Multipart middleware)' + requires :chart, type: ::API::Validations::Types::WorkhorseFile, desc: 'The chart file to be published (generated by Multipart middleware)', documentation: { type: 'file' } end post "api/:channel/charts" do authorize_upload!(authorized_user_project) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index e29d76a5950..0eb4fbb196c 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -18,6 +18,7 @@ module API API_TOKEN_ENV = 'gitlab.api.token' API_EXCEPTION_ENV = 'gitlab.api.exception' API_RESPONSE_STATUS_CODE = 'gitlab.api.response_status_code' + INTEGER_ID_REGEX = /^-?\d+$/.freeze def declared_params(options = {}) options = { include_parent_namespaces: false }.merge(options) @@ -139,7 +140,7 @@ module API projects = Project.without_deleted.not_hidden - if id.is_a?(Integer) || id =~ /^\d+$/ + if id.is_a?(Integer) || id =~ INTEGER_ID_REGEX projects.find_by(id: id) elsif id.include?("/") projects.find_by_full_path(id) @@ -168,7 +169,7 @@ module API # rubocop: disable CodeReuse/ActiveRecord def find_group(id) - if id.to_s =~ /^\d+$/ + if id.to_s =~ INTEGER_ID_REGEX Group.find_by(id: id) else Group.find_by_full_path(id) @@ -203,7 +204,7 @@ module API # rubocop: disable CodeReuse/ActiveRecord def find_namespace(id) - if id.to_s =~ /^\d+$/ + if id.to_s =~ INTEGER_ID_REGEX Namespace.without_project_namespaces.find_by(id: id) else find_namespace_by_path(id) @@ -286,22 +287,11 @@ module API end def authenticate_by_gitlab_shell_token! - if Feature.enabled?(:gitlab_shell_jwt_token) - begin - payload, _ = JSONWebToken::HMACToken.decode(headers[GITLAB_SHELL_API_HEADER], secret_token) - unauthorized! unless payload['iss'] == GITLAB_SHELL_JWT_ISSUER - rescue JWT::DecodeError, JWT::ExpiredSignature, JWT::ImmatureSignature => ex - Gitlab::ErrorTracking.track_exception(ex) - unauthorized! - end - else - input = params['secret_token'] - input ||= Base64.decode64(headers[GITLAB_SHARED_SECRET_HEADER]) if headers.key?(GITLAB_SHARED_SECRET_HEADER) - - input&.chomp! - - unauthorized! unless Devise.secure_compare(secret_token, input) - end + payload, _ = JSONWebToken::HMACToken.decode(headers[GITLAB_SHELL_API_HEADER], secret_token) + unauthorized! unless payload['iss'] == GITLAB_SHELL_JWT_ISSUER + rescue JWT::DecodeError, JWT::ExpiredSignature, JWT::ImmatureSignature => ex + Gitlab::ErrorTracking.track_exception(ex) + unauthorized! end def authenticated_with_can_read_all_resources! @@ -602,19 +592,19 @@ module API end end - def present_artifacts_file!(file, **args) + def present_artifacts_file!(file, project:, **args) log_artifacts_filesize(file&.model) - present_carrierwave_file!(file, **args) + present_carrierwave_file!(file, project: project, **args) end - def present_carrierwave_file!(file, supports_direct_download: true) + def present_carrierwave_file!(file, project: nil, supports_direct_download: true) return not_found! unless file&.exists? if file.file_storage? present_disk_file!(file.path, file.filename) elsif supports_direct_download && file.class.direct_download_enabled? - redirect(file.url) + redirect(cdn_fronted_url(file, project)) else header(*Gitlab::Workhorse.send_url(file.url)) status :ok @@ -622,6 +612,16 @@ module API end end + def cdn_fronted_url(file, project) + if file.respond_to?(:cdn_enabled_url) + result = file.cdn_enabled_url(project, ip_address) + Gitlab::ApplicationContext.push(artifact_used_cdn: result.used_cdn) + result.url + else + file.url + end + end + def increment_counter(event_name) Gitlab::UsageDataCounters.count(event_name) rescue StandardError => error @@ -732,13 +732,7 @@ module API end def secret_token - if Feature.enabled?(:gitlab_shell_jwt_token) - strong_memoize(:secret_token) do - File.read(Gitlab.config.gitlab_shell.secret_file) - end - else - Gitlab::Shell.secret_token - end + Gitlab::Shell.secret_token end def authenticate_non_public? diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb index e9af50b80be..74c8b582fde 100644 --- a/lib/api/helpers/groups_helpers.rb +++ b/lib/api/helpers/groups_helpers.rb @@ -11,7 +11,7 @@ module API optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the group' - optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for the group' + optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for the group', documentation: { type: 'file' } optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users in this group to setup Two-factor authentication' optional :two_factor_grace_period, type: Integer, desc: 'Time before Two-factor authentication is enforced' diff --git a/lib/api/helpers/open_api.rb b/lib/api/helpers/open_api.rb new file mode 100644 index 00000000000..11602244b57 --- /dev/null +++ b/lib/api/helpers/open_api.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module API + module Helpers + module OpenApi + extend ActiveSupport::Concern + + class_methods do + def add_open_api_documentation! + return if Rails.env.production? + + open_api_config = YAML.load_file(Rails.root.join('config/open_api.yml'))['metadata'].deep_symbolize_keys + + add_swagger_documentation(open_api_config) + end + end + end + end +end diff --git a/lib/api/helpers/packages/basic_auth_helpers.rb b/lib/api/helpers/packages/basic_auth_helpers.rb index ebedb3b7563..a62bb1d4991 100644 --- a/lib/api/helpers/packages/basic_auth_helpers.rb +++ b/lib/api/helpers/packages/basic_auth_helpers.rb @@ -14,15 +14,27 @@ module API include Constants include Gitlab::Utils::StrongMemoize - def authorized_user_project - @authorized_user_project ||= authorized_project_find! + def authorized_user_project(action: :read_project) + strong_memoize("authorized_user_project_#{action}") do + authorized_project_find!(action: action) + end end - def authorized_project_find! + def authorized_project_find!(action: :read_project) project = find_project(params[:id]) - unless project && can?(current_user, :read_project, project) - return unauthorized_or! { not_found! } + return unauthorized_or! { not_found! } unless project + + case action + when :read_package + unless can?(current_user, :read_package, project&.packages_policy_subject) + # guest users can have :read_project but not :read_package + return forbidden! if can?(current_user, :read_project, project) + + return unauthorized_or! { not_found! } + end + else + return unauthorized_or! { not_found! } unless can?(current_user, action, project) end project diff --git a/lib/api/helpers/packages/dependency_proxy_helpers.rb b/lib/api/helpers/packages/dependency_proxy_helpers.rb index a09499e00d7..dc81e5e1b51 100644 --- a/lib/api/helpers/packages/dependency_proxy_helpers.rb +++ b/lib/api/helpers/packages/dependency_proxy_helpers.rb @@ -16,8 +16,8 @@ module API maven: 'maven_package_requests_forwarding' }.freeze - def redirect_registry_request(forward_to_registry, package_type, options) - if forward_to_registry && redirect_registry_request_available?(package_type) && maven_forwarding_ff_enabled?(package_type, options[:target]) + def redirect_registry_request(forward_to_registry: false, package_type: nil, target: nil, **options) + if forward_to_registry && redirect_registry_request_available?(package_type, target) && maven_forwarding_ff_enabled?(package_type, target) ::Gitlab::Tracking.event(self.options[:for].name, "#{package_type}_request_forward") redirect(registry_url(package_type, options)) else @@ -40,15 +40,19 @@ module API end end - def redirect_registry_request_available?(package_type) + def redirect_registry_request_available?(package_type, target) application_setting_name = APPLICATION_SETTING_NAMES[package_type] raise ArgumentError, "Can't find application setting for package_type #{package_type}" unless application_setting_name - ::Gitlab::CurrentSettings - .current_application_settings - .attributes - .fetch(application_setting_name, false) + if target.present? && Feature.enabled?(:cascade_package_forwarding_settings, target) + target.public_send(application_setting_name) # rubocop:disable GitlabSecurity/PublicSend + else + ::Gitlab::CurrentSettings + .current_application_settings + .attributes + .fetch(application_setting_name, false) + end end private diff --git a/lib/api/helpers/personal_access_tokens_helpers.rb b/lib/api/helpers/personal_access_tokens_helpers.rb index db28daa5396..4fd72d89f4c 100644 --- a/lib/api/helpers/personal_access_tokens_helpers.rb +++ b/lib/api/helpers/personal_access_tokens_helpers.rb @@ -4,11 +4,14 @@ module API module Helpers module PersonalAccessTokensHelpers def finder_params(current_user) - if current_user.can_admin_all_resources? - { user: user(params[:user_id]) } - else - { user: current_user, impersonation: false } - end + user_param = + if current_user.can_admin_all_resources? + { user: user(params[:user_id]) } + else + { user: current_user, impersonation: false } + end + + declared(params, include_missing: false).merge(user_param) end def user(user_id) diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 7ca3f55b5a2..9839828a5b4 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -36,6 +36,7 @@ module API optional :analytics_access_level, type: String, values: %w(disabled private enabled), desc: 'Analytics access level. One of `disabled`, `private` or `enabled`' optional :container_registry_access_level, type: String, values: %w(disabled private enabled), desc: 'Controls visibility of the container registry. One of `disabled`, `private` or `enabled`. `private` will make the container registry accessible only to project members (reporter role and above). `enabled` will make the container registry accessible to everyone who has access to the project. `disabled` will disable the container registry' optional :security_and_compliance_access_level, type: String, values: %w(disabled private enabled), desc: 'Security and compliance access level. One of `disabled`, `private` or `enabled`' + optional :releases_access_level, type: String, values: %w(disabled private enabled), desc: 'Releases access level. One of `disabled`, `private` or `enabled`' optional :emails_disabled, type: Boolean, desc: 'Disable email notifications' optional :show_default_award_emojis, type: Boolean, desc: 'Show default award emojis' @@ -58,7 +59,7 @@ module API optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all threads are resolved' optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Deprecated: Use :topics instead' optional :topics, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The list of topics for a project' - optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for project' + optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for project', documentation: { type: 'file' } optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' optional :suggestion_commit_message, type: String, desc: 'The commit message used to apply merge request suggestions' @@ -72,7 +73,7 @@ module API optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins' optional :packages_enabled, type: Boolean, desc: 'Enable project packages feature' optional :squash_option, type: String, values: %w(never always default_on default_off), desc: 'Squash default for project. One of `never`, `always`, `default_on`, or `default_off`.' - optional :mr_default_target_self, Boolean, desc: 'Merge requests of this forked project targets itself by default' + optional :mr_default_target_self, type: Boolean, desc: 'Merge requests of this forked project targets itself by default' end params :optional_project_params_ee do @@ -179,6 +180,7 @@ module API :keep_latest_artifact, :mr_default_target_self, :enforce_auth_checks_on_uploads, + :releases_access_level, # TODO: remove in API v5, replaced by *_access_level :issues_enabled, diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb index 46ca8e4c428..493cc038f46 100644 --- a/lib/api/import_github.rb +++ b/lib/api/import_github.rb @@ -43,6 +43,7 @@ module API optional :new_name, type: String, desc: 'New repo name' requires :target_namespace, type: String, desc: 'Namespace to import repo into' optional :github_hostname, type: String, desc: 'Custom GitHub enterprise hostname' + optional :optional_stages, type: Hash, desc: 'Optional stages of import to be performed' end post 'import/github' do result = Import::GithubService.new(client, current_user, params).execute(access_params, provider) @@ -54,5 +55,20 @@ module API { errors: result[:message] } end end + + params do + requires :project_id, type: Integer, desc: 'ID of importing project to be canceled' + end + post 'import/github/cancel' do + project = Project.imported_from(provider.to_s).find(params[:project_id]) + result = Import::Github::CancelProjectImportService.new(project, current_user).execute + + if result[:status] == :success + status :ok + present ProjectSerializer.new.represent(project, serializer: :import) + else + render_api_error!(result[:message], result[:http_status]) + end + end end end diff --git a/lib/api/internal/pages.rb b/lib/api/internal/pages.rb index 20ca7038471..6be2679af14 100644 --- a/lib/api/internal/pages.rb +++ b/lib/api/internal/pages.rb @@ -59,7 +59,8 @@ module API # Gitlab::Pages::CacheControl present_cached virtual_domain, cache_context: nil, - with: Entities::Internal::Pages::VirtualDomain + with: Entities::Internal::Pages::VirtualDomain, + expires_in: ::Gitlab::Pages::CacheControl::EXPIRE else present virtual_domain, with: Entities::Internal::Pages::VirtualDomain end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index b6ad34424a6..b8b4019765d 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -272,17 +272,21 @@ module API begin spam_params = ::Spam::SpamParams.new_from_request(request: request) - issue = ::Issues::CreateService.new(project: user_project, - current_user: current_user, - params: issue_params, - spam_params: spam_params).execute + result = ::Issues::CreateService.new(project: user_project, + current_user: current_user, + params: issue_params, + spam_params: spam_params).execute + + if result.success? + present result[:issue], with: Entities::Issue, current_user: current_user, project: user_project + elsif result[:issue] + issue = result[:issue] - if issue.valid? - present issue, with: Entities::Issue, current_user: current_user, project: user_project - else with_captcha_check_rest_api(spammable: issue) do render_validation_error!(issue) end + else + render_api_error!(result.errors.join(', '), result.http_status || 422) end rescue ::ActiveRecord::RecordNotUnique render_api_error!('Duplicated issue', 409) diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index a3a25ec1696..72313d6a588 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -125,7 +125,13 @@ module API no_package_found = package_file ? false : true - redirect_registry_request(no_package_found, :maven, path: params[:path], file_name: params[:file_name], target: params[:target]) do + redirect_registry_request( + forward_to_registry: no_package_found, + package_type: :maven, + target: params[:target], + path: params[:path], + file_name: params[:file_name] + ) do not_found!('Package') if no_package_found case format diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 1dc0e1f0d22..a0e7d0b10cd 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -218,6 +218,7 @@ module API [ current_user&.cache_key, mr.merge_status, + mr.labels.map(&:cache_key), mr.merge_request_assignees.map(&:cache_key), mr.merge_request_reviewers.map(&:cache_key) ].join(":") @@ -560,7 +561,7 @@ module API put ':id/merge_requests/:merge_request_iid/reset_approvals', feature_category: :code_review, urgency: :low do merge_request = find_project_merge_request(params[:merge_request_iid]) - unauthorized! unless current_user.bot? && merge_request.can_be_approved_by?(current_user) + unauthorized! unless current_user.bot? && merge_request.eligible_for_approval_by?(current_user) merge_request.approvals.delete_all diff --git a/lib/api/metadata.rb b/lib/api/metadata.rb index c4984f0e7f0..3e42ffe336a 100644 --- a/lib/api/metadata.rb +++ b/lib/api/metadata.rb @@ -25,15 +25,76 @@ module API } EOF - desc 'Get the metadata information of the GitLab instance.' do + helpers do + def run_metadata_query + run_graphql!( + query: METADATA_QUERY, + context: { current_user: current_user }, + transform: ->(result) { result.dig('data', 'metadata') } + ) + end + end + + desc 'Retrieve metadata information for this GitLab instance.' do detail 'This feature was introduced in GitLab 15.2.' + success [ + { + code: 200, + model: Entities::Metadata, + message: 'successful operation', + examples: { + successful_response: { + 'value' => { + version: "15.0-pre", + revision: "c401a659d0c", + kas: { + enabled: true, + externalUrl: "grpc://gitlab.example.com:8150", + version: "15.0.0" + } + } + } + } + } + ] + failure [{ code: 401, message: 'unauthorized operation' }] + tags %w[metadata] end get '/metadata' do - run_graphql!( - query: METADATA_QUERY, - context: { current_user: current_user }, - transform: ->(result) { result.dig('data', 'metadata') } - ) + run_metadata_query + end + + # Support the deprecated `/version` route. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/366287 + desc 'Get the version information of the GitLab instance.' do + detail 'This feature was introduced in GitLab 8.13 and deprecated in 15.5. ' \ + 'We recommend you instead use the Metadata API.' + success [ + { + code: 200, + model: Entities::Metadata, + message: 'successful operation', + examples: { + 'Example' => { + 'value' => { + version: "15.0-pre", + revision: "c401a659d0c", + kas: { + enabled: true, + externalUrl: "grpc://gitlab.example.com:8150", + version: "15.0.0" + } + } + } + } + } + ] + failure [{ code: 401, message: 'unauthorized operation' }] + tags %w[metadata] + end + + get '/version' do + run_metadata_query end end end diff --git a/lib/api/ml/mlflow.rb b/lib/api/ml/mlflow.rb index 4f5bd42f8f9..2ffb04ebcbd 100644 --- a/lib/api/ml/mlflow.rb +++ b/lib/api/ml/mlflow.rb @@ -9,20 +9,28 @@ module API include APIGuard # The first part of the url is the namespace, the second part of the URL is what the MLFlow client calls - MLFLOW_API_PREFIX = ':id/ml/mflow/api/2.0/mlflow/' + MLFLOW_API_PREFIX = ':id/ml/mlflow/api/2.0/mlflow/' allow_access_with_scope :api allow_access_with_scope :read_api, if: -> (request) { request.get? || request.head? } + feature_category :mlops + + content_type :json, 'application/json' + default_format :json + before do + # MLFlow Client considers any status code different than 200 an error, even 201 + status 200 + authenticate! + not_found! unless Feature.enabled?(:ml_experiment_tracking, user_project) end - feature_category :mlops - - content_type :json, 'application/json' - default_format :json + rescue_from ActiveRecord::ActiveRecordError do |e| + invalid_parameter!(e.message) + end helpers do def resource_not_found! @@ -32,6 +40,34 @@ module API def resource_already_exists! render_structured_api_error!({ error_code: 'RESOURCE_ALREADY_EXISTS' }, 400) end + + def invalid_parameter!(message = nil) + render_structured_api_error!({ error_code: 'INVALID_PARAMETER_VALUE', message: message }, 400) + end + + def experiment_repository + ::Ml::ExperimentTracking::ExperimentRepository.new(user_project, current_user) + end + + def candidate_repository + ::Ml::ExperimentTracking::CandidateRepository.new(user_project, current_user) + end + + def experiment + @experiment ||= find_experiment!(params[:experiment_id], params[:experiment_name]) + end + + def candidate + @candidate ||= find_candidate!(params[:run_id]) + end + + def find_experiment!(iid, name) + experiment_repository.by_iid_or_name(iid: iid, name: name) || resource_not_found! + end + + def find_candidate!(iid) + candidate_repository.by_iid(iid) || resource_not_found! + end end params do @@ -44,33 +80,35 @@ module API namespace MLFLOW_API_PREFIX do resource :experiments do desc 'Fetch experiment by experiment_id' do - success Entities::Ml::Mlflow::Experiment + success Entities::Ml::Mlflow::GetExperiment detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-experiment' end params do optional :experiment_id, type: String, default: '', desc: 'Experiment ID, in reference to the project' end get 'get', urgency: :low do - experiment = ::Ml::Experiment.by_project_id_and_iid(user_project.id, params[:experiment_id]) - - resource_not_found! unless experiment - - present experiment, with: Entities::Ml::Mlflow::Experiment + present experiment, with: Entities::Ml::Mlflow::GetExperiment end desc 'Fetch experiment by experiment_name' do - success Entities::Ml::Mlflow::Experiment + success Entities::Ml::Mlflow::GetExperiment detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-experiment-by-name' end params do optional :experiment_name, type: String, default: '', desc: 'Experiment name' end get 'get-by-name', urgency: :low do - experiment = ::Ml::Experiment.by_project_id_and_name(user_project, params[:experiment_name]) + present experiment, with: Entities::Ml::Mlflow::GetExperiment + end - resource_not_found! unless experiment + desc 'List experiments' do + success Entities::Ml::Mlflow::ListExperiment + detail 'https://www.mlflow.org/docs/latest/rest-api.html#list-experiments' + end + get 'list', urgency: :low do + response = { experiments: experiment_repository.all } - present experiment, with: Entities::Ml::Mlflow::Experiment + present response, with: Entities::Ml::Mlflow::ListExperiment end desc 'Create experiment' do @@ -83,33 +121,13 @@ module API optional :tags, type: Array, desc: 'This will be ignored' end post 'create', urgency: :low do - resource_already_exists! if ::Ml::Experiment.has_record?(user_project.id, params[:name]) - - experiment = ::Ml::Experiment.create!(name: params[:name], - user: current_user, - project: user_project) - - present experiment, with: Entities::Ml::Mlflow::NewExperiment + present experiment_repository.create!(params[:name]), with: Entities::Ml::Mlflow::NewExperiment + rescue ActiveRecord::RecordInvalid + resource_already_exists! end end resource :runs do - desc 'Gets an MLFlow Run, which maps to GitLab Candidates' do - success Entities::Ml::Mlflow::Run - detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-run' - end - params do - optional :run_id, type: String, desc: 'UUID of the candidate.' - optional :run_uuid, type: String, desc: 'This parameter is ignored' - end - get 'get', urgency: :low do - candidate = ::Ml::Candidate.with_project_id_and_iid(user_project.id, params[:run_id]) - - resource_not_found! unless candidate - - present candidate, with: Entities::Ml::Mlflow::Run - end - desc 'Creates a Run.' do success Entities::Ml::Mlflow::Run detail ['https://www.mlflow.org/docs/1.28.0/rest-api.html#create-run', @@ -125,16 +143,18 @@ module API optional :tags, type: Array, desc: 'This will be ignored' end post 'create', urgency: :low do - experiment = ::Ml::Experiment.by_project_id_and_iid(user_project.id, params[:experiment_id].to_i) - - resource_not_found! unless experiment - - candidate = ::Ml::Candidate.create!( - experiment: experiment, - user: current_user, - start_time: params[:start_time] || 0 - ) + present candidate_repository.create!(experiment, params[:start_time]), with: Entities::Ml::Mlflow::Run + end + desc 'Gets an MLFlow Run, which maps to GitLab Candidates' do + success Entities::Ml::Mlflow::Run + detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-run' + end + params do + requires :run_id, type: String, desc: 'UUID of the candidate.' + optional :run_uuid, type: String, desc: 'This parameter is ignored' + end + get 'get', urgency: :low do present candidate, with: Entities::Ml::Mlflow::Run end @@ -144,7 +164,7 @@ module API 'MLFlow Runs map to GitLab Candidates'] end params do - optional :run_id, type: String, desc: 'UUID of the candidate.' + requires :run_id, type: String, desc: 'UUID of the candidate.' optional :status, type: String, values: ::Ml::Candidate.statuses.keys.map(&:upcase), desc: "Status of the run. Accepts: " \ @@ -152,16 +172,79 @@ module API optional :end_time, type: Integer, desc: 'Ending time of the run' end post 'update', urgency: :low do - candidate = ::Ml::Candidate.with_project_id_and_iid(user_project.id, params[:run_id]) + candidate_repository.update(candidate, params[:status], params[:end_time]) - resource_not_found! unless candidate + present candidate, with: Entities::Ml::Mlflow::UpdateRun + end - candidate.status = params[:status].downcase if params[:status] - candidate.end_time = params[:end_time] if params[:end_time] + desc 'Logs a metric to a run.' do + summary 'Log a metric for a run. A metric is a key-value pair (string key, float value) with an '\ + 'associated timestamp. Examples include the various metrics that represent ML model accuracy. '\ + 'A metric can be logged multiple times.' + detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-metric' + end + params do + requires :run_id, type: String, desc: 'UUID of the run.' + requires :key, type: String, desc: 'Name for the metric.' + requires :value, type: Float, desc: 'Value of the metric.' + requires :timestamp, type: Integer, desc: 'Unix timestamp in milliseconds when metric was recorded' + optional :step, type: Integer, desc: 'Step at which the metric was recorded' + end + post 'log-metric', urgency: :low do + candidate_repository.add_metric!( + candidate, + params[:key], + params[:value], + params[:timestamp], + params[:step] + ) + + {} + end - candidate.save if candidate.valid? + desc 'Logs a parameter to a run.' do + summary 'Log a param used for a run. A param is a key-value pair (string key, string value). '\ + 'Examples include hyperparameters used for ML model training and constant dates and values '\ + 'used in an ETL pipeline. A param can be logged only once for a run, duplicate will be .'\ + 'ignored' - present candidate, with: Entities::Ml::Mlflow::UpdateRun + detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-param' + end + params do + requires :run_id, type: String, desc: 'UUID of the run.' + requires :key, type: String, desc: 'Name for the parameter.' + requires :value, type: String, desc: 'Value for the parameter.' + end + post 'log-parameter', urgency: :low do + bad_request! unless candidate_repository.add_param!(candidate, params[:key], params[:value]) + + {} + end + + desc 'Logs multiple parameters and metrics.' do + summary 'Log a batch of metrics and params for a run. Validation errors will block the entire batch, '\ + 'duplicate errors will be ignored.' + + detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-param' + end + params do + requires :run_id, type: String, desc: 'UUID of the run.' + optional :metrics, type: Array, default: [] do + requires :key, type: String, desc: 'Name for the metric.' + requires :value, type: Float, desc: 'Value of the metric.' + requires :timestamp, type: Integer, desc: 'Unix timestamp in milliseconds when metric was recorded' + optional :step, type: Integer, desc: 'Step at which the metric was recorded' + end + optional :params, type: Array, default: [] do + requires :key, type: String, desc: 'Name for the metric.' + requires :value, type: String, desc: 'Value of the metric.' + end + end + post 'log-batch', urgency: :low do + candidate_repository.add_metrics(candidate, params[:metrics]) + candidate_repository.add_params(candidate, params[:params]) + + {} end end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 77c479c529a..8ce875cdc03 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -73,7 +73,7 @@ module API params do requires :noteable_id, type: Integer, desc: 'The ID of the noteable' requires :body, type: String, desc: 'The content of a note' - optional :confidential, type: Boolean, desc: '[Deprecated in 15.3] Renamed to internal' + optional :confidential, type: Boolean, desc: '[Deprecated in 15.5] Renamed to internal' optional :internal, type: Boolean, desc: 'Internal note flag, default is false' optional :created_at, type: String, desc: 'The creation date of the note' optional :merge_request_diff_head_sha, type: String, desc: 'The SHA of the head commit' diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index 34d3a5150da..9cf61967ba4 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -106,7 +106,9 @@ module API pages_domain_params = declared(params, include_parent_namespaces: false) - pages_domain = user_project.pages_domains.create(pages_domain_params) + pages_domain = ::PagesDomains::CreateService + .new(user_project, current_user, pages_domain_params) + .execute if pages_domain.persisted? present pages_domain, with: Entities::PagesDomain @@ -136,7 +138,9 @@ module API pages_domain_params.delete(:user_provided_key) end - if pages_domain.update(pages_domain_params) + service = ::PagesDomains::UpdateService.new(user_project, current_user, pages_domain_params) + + if service.execute(pages_domain) present pages_domain, with: Entities::PagesDomain else render_validation_error!(pages_domain) @@ -150,7 +154,9 @@ module API delete ":id/pages/domains/:domain", requirements: PAGES_DOMAINS_ENDPOINT_REQUIREMENTS do authorize! :update_pages, user_project - pages_domain.destroy + ::PagesDomains::DeleteService + .new(user_project, current_user) + .execute(pages_domain) no_content! end diff --git a/lib/api/personal_access_tokens.rb b/lib/api/personal_access_tokens.rb index 1c00569bba2..a2903faa4ad 100644 --- a/lib/api/personal_access_tokens.rb +++ b/lib/api/personal_access_tokens.rb @@ -11,7 +11,15 @@ module API success Entities::PersonalAccessToken end params do - optional :user_id, type: Integer, desc: 'User ID' + optional :user_id, type: Integer, desc: 'Filter PATs by User ID' + optional :revoked, type: Boolean, desc: 'Filter PATs where revoked state matches parameter' + optional :state, type: String, desc: 'Filter PATs which are either active or not', + values: %w[active inactive] + optional :created_before, type: DateTime, desc: 'Filter PATs which were created before given datetime' + optional :created_after, type: DateTime, desc: 'Filter PATs which were created after given datetime' + optional :last_used_before, type: DateTime, desc: 'Filter PATs which were used before given datetime' + optional :last_used_after, type: DateTime, desc: 'Filter PATs which were used after given datetime' + optional :search, type: String, desc: 'Filters PATs by its name' use :pagination end diff --git a/lib/api/personal_access_tokens/self_revocation.rb b/lib/api/personal_access_tokens/self_information.rb index 22e07f4cc7b..89850614f94 100644 --- a/lib/api/personal_access_tokens/self_revocation.rb +++ b/lib/api/personal_access_tokens/self_information.rb @@ -2,21 +2,25 @@ module API class PersonalAccessTokens - class SelfRevocation < ::API::Base + class SelfInformation < ::API::Base include APIGuard feature_category :authentication_and_authorization helpers ::API::Helpers::PersonalAccessTokensHelpers - # As any token regardless of `scope` should be able to revoke itself - # all availabe scopes are allowed for this API class. + # As any token regardless of `scope` should be able to view/revoke itself + # all available scopes are allowed for this API class. # Please be aware of the permissive scope when adding new endpoints to this class. allow_access_with_scope(Gitlab::Auth.all_available_scopes) before { authenticate! } resource :personal_access_tokens do + get 'self' do + present access_token, with: Entities::PersonalAccessToken + end + delete 'self' do revoke_token(access_token) end diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index d610b5e4f95..29fdfe45566 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -46,7 +46,8 @@ module API optional :description, type: String, desc: 'Override the project description' optional :upload, type: Hash do optional :url, type: String, desc: 'The URL to upload the project' - optional :http_method, type: String, default: 'PUT', desc: 'HTTP method to upload the exported project' + optional :http_method, type: String, default: 'PUT', values: %w[PUT POST], + desc: 'HTTP method to upload the exported project' end end post ':id/export' do diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 7a66044c5b6..0da8c1ecedd 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -55,7 +55,7 @@ module API params do requires :path, type: String, desc: 'The new project path and name' - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The project export file to be imported' + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The project export file to be imported', documentation: { type: 'file' } optional :name, type: String, desc: 'The name of the project to be imported. Defaults to the path of the project if not provided.' optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace." optional :overwrite, type: Boolean, default: false, desc: 'If there is a project in the same namespace and with the same name overwrite it' diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 8c58cc585d8..bb97f4fa7ce 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -375,7 +375,7 @@ module API optional :name, type: String, desc: 'The name that will be assigned to the fork' optional :description, type: String, desc: 'The description that will be assigned to the fork' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the fork' - optional :mr_default_target_self, Boolean, desc: 'Merge requests of this forked project targets itself by default' + optional :mr_default_target_self, type: Boolean, desc: 'Merge requests of this forked project targets itself by default' end post ':id/fork', feature_category: :source_code_management do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20759') diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index ae583ca968a..1f27fcce879 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -56,7 +56,12 @@ module API packages = Packages::Pypi::PackagesFinder.new(current_user, group_or_project, { package_name: params[:package_name] }).execute empty_packages = packages.empty? - redirect_registry_request(empty_packages, :pypi, package_name: params[:package_name]) do + redirect_registry_request( + forward_to_registry: empty_packages, + package_type: :pypi, + target: group_or_project, + package_name: params[:package_name] + ) do not_found!('Package') if empty_packages presenter = ::Packages::Pypi::SimplePackageVersionsPresenter.new(packages, group_or_project) diff --git a/lib/api/search.rb b/lib/api/search.rb index 44bb4228786..ff17696ed3e 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -63,7 +63,7 @@ module API @results = search_service(additional_params).search_objects(preload_method) end - set_global_search_log_information + set_global_search_log_information(additional_params) Gitlab::Metrics::GlobalSearchSlis.record_apdex( elapsed: @search_duration_s, @@ -105,7 +105,7 @@ module API # EE, without having to modify this file directly. end - def search_type + def search_type(additional_params = {}) 'basic' end @@ -113,10 +113,10 @@ module API params[:scope] end - def set_global_search_log_information + def set_global_search_log_information(additional_params) Gitlab::Instrumentation::GlobalSearchApi.set_information( - type: search_type, - level: search_service.level, + type: search_type(additional_params), + level: search_service(additional_params).level, scope: search_scope, search_duration_s: @search_duration_s ) diff --git a/lib/api/settings.rb b/lib/api/settings.rb index f393f862f55..8c8b6c0a1ba 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -132,7 +132,7 @@ module API requires :recaptcha_private_key, type: String, desc: 'Generate private key at http://www.google.com/recaptcha' end optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." - optional :repository_storages_weighted, type: Hash, coerce_with: Validations::Types::HashOfIntegerValues.coerce, desc: 'Storage paths for new projects with a weighted value ranging from 0 to 100' + optional :repository_storages_weighted, type: Hash, coerce_with: Validations::Types::HashOfIntegerValues.coerce, desc: 'Storage paths for new projects with a weighted value ranging from 0 to 100', documentation: { type: 'Object', additional_properties: Integer } optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to set up Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 4e70ebddf94..5f8e6c806cb 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -6,6 +6,7 @@ module API include PaginationParams feature_category :snippets + urgency :low resource :snippets do helpers Helpers::SnippetsHelpers @@ -51,7 +52,7 @@ module API use :pagination end - get 'public', urgency: :low do + get 'public' do authenticate! filter_params = declared_params(include_missing: false).merge(only_personal: true) @@ -192,7 +193,7 @@ module API params do use :raw_file_params end - get ":id/files/:ref/:file_path/raw", urgency: :low, requirements: { file_path: API::NO_SLASH_URL_PART_REGEX } do + get ":id/files/:ref/:file_path/raw", requirements: { file_path: API::NO_SLASH_URL_PART_REGEX } do snippet = snippets.find_by_id(params.delete(:id)) not_found!('Snippet') unless snippet&.repo_exists? diff --git a/lib/api/support/git_access_actor.rb b/lib/api/support/git_access_actor.rb index f450630afdd..16861a146ae 100644 --- a/lib/api/support/git_access_actor.rb +++ b/lib/api/support/git_access_actor.rb @@ -57,3 +57,5 @@ module API end end end + +API::Support::GitAccessActor.prepend_mod_with('API::Support::GitAccessActor') diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 85a299c5673..a80ef514943 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -7,15 +7,18 @@ module API GLOBAL_TEMPLATE_TYPES = { gitignores: { gitlab_version: 8.8, - feature_category: :source_code_management + feature_category: :source_code_management, + file_type: '.gitignore' }, gitlab_ci_ymls: { gitlab_version: 8.9, - feature_category: :pipeline_authoring + feature_category: :pipeline_authoring, + file_type: 'GitLab CI/CD YAML' }, dockerfiles: { gitlab_version: 8.15, - feature_category: :source_code_management + feature_category: :source_code_management, + file_type: 'Dockerfile' } }.freeze @@ -26,7 +29,7 @@ module API end end - desc 'Get the list of the available license template' do + desc 'Get all license templates' do detail 'This feature was introduced in GitLab 8.7.' success ::API::Entities::License end @@ -43,12 +46,14 @@ module API present paginate(::Kaminari.paginate_array(templates)), with: ::API::Entities::License end - desc 'Get the text for a specific license' do + desc 'Get a single license template' do detail 'This feature was introduced in GitLab 8.7.' success ::API::Entities::License end params do - requires :name, type: String, desc: 'The name of the template' + requires :name, type: String, desc: 'The name of the license template' + optional :project, type: String, desc: 'The copyrighted project name' + optional :fullname, type: String, desc: 'The full-name of the copyright holder' end get "templates/licenses/:name", requirements: { name: /[\w\.-]+/ }, feature_category: :source_code_management do template = TemplateFinder.build(:licenses, nil, name: params[:name]).execute @@ -65,8 +70,9 @@ module API GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| gitlab_version = properties[:gitlab_version] + file_type = properties[:file_type] - desc 'Get the list of the available template' do + desc "Get all #{file_type} templates" do detail "This feature was introduced in GitLab #{gitlab_version}." success Entities::TemplatesList end @@ -78,12 +84,12 @@ module API present paginate(templates), with: Entities::TemplatesList end - desc 'Get the text for a specific template present in local filesystem' do + desc "Get a single #{file_type} template" do detail "This feature was introduced in GitLab #{gitlab_version}." success Entities::Template end params do - requires :name, type: String, desc: 'The name of the template' + requires :name, type: String, desc: "The name of the #{file_type} template" end get "templates/#{template_type}/:name", requirements: { name: /[\w\.-]+/ }, feature_category: properties[:feature_category] do finder = TemplateFinder.build(template_type, nil, name: declared(params)[:name]) diff --git a/lib/api/todos.rb b/lib/api/todos.rb index f1779df7cc6..57745ee8802 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -15,17 +15,17 @@ module API }.freeze params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, type: String, desc: 'The ID or URL-encoded path of the project owned by the authenticated user' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do ISSUABLE_TYPES.each do |type, finder| type_id_str = "#{type.singularize}_iid".to_sym - desc 'Create a todo on an issuable' do + desc 'Create a to-do item on an issuable' do success Entities::Todo end params do - requires type_id_str, type: Integer, desc: 'The IID of an issuable' + requires type_id_str, type: Integer, desc: 'The internal ID of an issuable' end post ":id/#{type}/:#{type_id_str}/todo" do issuable = instance_exec(params[type_id_str], &finder) @@ -44,12 +44,12 @@ module API resource :todos do helpers do params :todo_filters do - optional :action, String, values: Todo::ACTION_NAMES.values.map(&:to_s) - optional :author_id, Integer - optional :state, String, values: Todo.state_machine.states.map(&:name).map(&:to_s) - optional :type, String, values: TodosFinder.todo_types - optional :project_id, Integer - optional :group_id, Integer + optional :action, type: String, values: Todo::ACTION_NAMES.values.map(&:to_s), desc: 'The action to be filtered' + optional :author_id, type: Integer, desc: 'The ID of an author' + optional :project_id, type: Integer, desc: 'The ID of a project' + optional :group_id, type: Integer, desc: 'The ID of a group' + optional :state, type: String, values: Todo.state_machine.states.map(&:name).map(&:to_s), desc: 'The state of the to-do item' + optional :type, type: String, values: TodosFinder.todo_types.map(&:to_s), desc: 'The type of to-do item' end def find_todos @@ -81,7 +81,7 @@ module API end end - desc 'Get a todo list' do + desc 'Get a list of to-do items' do success Entities::Todo end params do @@ -96,11 +96,11 @@ module API present todos, options end - desc 'Mark a todo as done' do + desc 'Mark a to-do item as done' do success Entities::Todo end params do - requires :id, type: Integer, desc: 'The ID of the todo being marked as done' + requires :id, type: Integer, desc: 'The ID of to-do item' end post ':id/mark_as_done' do todo = current_user.todos.find(params[:id]) @@ -110,7 +110,7 @@ module API present todo, with: Entities::Todo, current_user: current_user end - desc 'Mark all todos as done' + desc 'Mark all to-do items as done' post '/mark_as_done' do todos = find_todos diff --git a/lib/api/users.rb b/lib/api/users.rb index 1d1c633824e..7f44e46f1ca 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -50,11 +50,13 @@ module API optional :provider, type: String, desc: 'The external provider' optional :bio, type: String, desc: 'The biography of the user' optional :location, type: String, desc: 'The location of the user' + optional :pronouns, type: String, desc: 'The pronouns of the user' optional :public_email, type: String, desc: 'The public email of the user' + optional :commit_email, type: String, desc: 'The commit email, _private for private commit email' optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator' optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' - optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for user' + optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for user', documentation: { type: 'file' } optional :theme_id, type: Integer, desc: 'The GitLab theme for the user' optional :color_scheme_id, type: Integer, desc: 'The color scheme for the file viewer' optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile' @@ -187,7 +189,10 @@ module API user = find_user(params[:id]) not_found!('User') unless user - if current_user.follow(user) + followee = current_user.follow(user) + if followee&.errors&.any? + render_api_error!(followee.errors.full_messages.join(', '), 400) + elsif followee&.persisted? present user, with: Entities::UserBasic else not_modified! @@ -885,7 +890,7 @@ module API params do requires :name, type: String, desc: 'The name of the impersonation token' optional :expires_at, type: Date, desc: 'The expiration date in the format YEAR-MONTH-DAY of the impersonation token' - optional :scopes, type: Array, desc: 'The array of scopes of the impersonation token' + optional :scopes, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The array of scopes of the impersonation token' end post feature_category: :authentication_and_authorization do impersonation_token = finder.build(declared_params(include_missing: false)) diff --git a/lib/api/version.rb b/lib/api/version.rb deleted file mode 100644 index bdce88ab827..00000000000 --- a/lib/api/version.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module API - class Version < ::API::Base - helpers ::API::Helpers::GraphqlHelpers - include APIGuard - - allow_access_with_scope :read_user, if: -> (request) { request.get? || request.head? } - - before { authenticate! } - - feature_category :not_owned # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned - - METADATA_QUERY = <<~EOF - { - metadata { - version - revision - } - } - EOF - - desc 'Get the version information of the GitLab instance.' do - detail 'This feature was introduced in GitLab 8.13.' - end - get '/version' do - run_graphql!( - query: METADATA_QUERY, - context: { current_user: current_user }, - transform: ->(result) { result.dig('data', 'metadata') } - ) - end - end -end diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index 082be1f7e11..bb8ad5c4285 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -133,7 +133,7 @@ module API success Entities::WikiAttachment end params do - requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The attachment file to be uploaded' + requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The attachment file to be uploaded', documentation: { type: 'file' } optional :branch, type: String, desc: 'The name of the branch' end post ":id/wikis/attachments" do diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb index cf368e28beb..7abfadc612b 100644 --- a/lib/banzai/filter/markdown_engines/common_mark.rb +++ b/lib/banzai/filter/markdown_engines/common_mark.rb @@ -56,3 +56,5 @@ module Banzai end end end + +Banzai::Filter::MarkdownEngines::CommonMark.prepend_mod diff --git a/lib/banzai/filter/pathological_markdown_filter.rb b/lib/banzai/filter/pathological_markdown_filter.rb deleted file mode 100644 index 0f94150c7a1..00000000000 --- a/lib/banzai/filter/pathological_markdown_filter.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - class PathologicalMarkdownFilter < HTML::Pipeline::TextFilter - # It's not necessary for this to be precise - we just need to detect - # when there are a non-trivial number of unclosed image links. - # So we don't really care about code blocks, etc. - # See https://gitlab.com/gitlab-org/gitlab/-/issues/370428 - REGEX = /!\[(?:[^\]])+?!\[/.freeze - DETECTION_MAX = 10 - - def call - count = 0 - - @text.scan(REGEX) do |_match| - count += 1 - break if count > DETECTION_MAX - end - - return @text if count <= DETECTION_MAX - - "_Unable to render markdown - too many unclosed markdown image links detected._" - end - end - end -end diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 179afd840a5..1c794a81d9d 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'cgi/util' + # Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/table_of_contents.js module Banzai module Filter @@ -93,7 +95,7 @@ module Banzai def text return '' unless node - @text ||= EscapeUtils.escape_html(node.text) + @text ||= CGI.escapeHTML(node.text) end private diff --git a/lib/banzai/filter/truncate_visible_filter.rb b/lib/banzai/filter/truncate_visible_filter.rb new file mode 100644 index 00000000000..edd6efd4706 --- /dev/null +++ b/lib/banzai/filter/truncate_visible_filter.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module Banzai + module Filter + class TruncateVisibleFilter < HTML::Pipeline::Filter + # Truncates the document to `truncate_visible_max_chars` characters, + # excluding any HTML tags. + + MATCH_CODE = 'pre > code > .line' + + def call + return doc unless context[:truncate_visible_max_chars].present? + + max_chars = context[:truncate_visible_max_chars] + content_length = 0 + @truncated = false + + doc.traverse do |node| + if node.text? || node.content.empty? + if truncated + node.remove + next + end + + handle_line_breaks(node) + truncate_content(content_length, max_chars, node) + + content_length += node.content.length + end + + truncate_if_block(node) + end + + doc + end + + private + + attr_reader :truncated + + def truncate_content(content_length, max_chars, node) + num_remaining = max_chars - content_length + return unless node.content.length > num_remaining + + node.content = node.content.truncate(num_remaining) + @truncated = true + end + + # Handle line breaks within a node + def handle_line_breaks(node) + return unless node.content.strip.lines.length > 1 + + node.content = "#{node.content.lines.first.chomp}..." + @truncated = true + end + + # If `node` is the first block element, and the + # text hasn't already been truncated, then append "..." to the node contents + # and return true. Otherwise return false. + def truncate_if_block(node) + return if truncated + return unless node.element? && (node.description&.block? || node.matches?(MATCH_CODE)) + + node.inner_html = "#{node.inner_html}..." if node.next_sibling + @truncated = true + end + end + end +end diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb index b4c2e7efae3..455cac98841 100644 --- a/lib/banzai/filter/wiki_link_filter/rewriter.rb +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -42,7 +42,7 @@ module Banzai def apply_relative_link_rules! if @uri.relative? && @uri.path.present? link = @uri.path - link = ::File.join(@wiki_base_path, link) unless link.starts_with?(@wiki_base_path) + link = ::File.join(@wiki_base_path, link) unless prefixed_with_base_path?(link) link = "#{link}##{@uri.fragment}" if @uri.fragment @uri = Addressable::URI.parse(link) end @@ -55,6 +55,15 @@ module Banzai def repository_upload? @uri.relative? && @uri.path.starts_with?(Wikis::CreateAttachmentService::ATTACHMENT_PATH) end + + def prefixed_with_base_path?(link) + link.starts_with?(@wiki_base_path) || link.starts_with?(old_wiki_base_path) + end + + # before we added `/-/` to all our paths + def old_wiki_base_path + @wiki_base_path.sub('/-/', '/') + end end end end diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb index 90edc7010f4..b652d8d89cf 100644 --- a/lib/banzai/pipeline/ascii_doc_pipeline.rb +++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb @@ -13,6 +13,7 @@ module Banzai Filter::ColorFilter, Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, + Filter::WikiLinkFilter, Filter::AsciiDocPostProcessingFilter ] end diff --git a/lib/banzai/pipeline/plain_markdown_pipeline.rb b/lib/banzai/pipeline/plain_markdown_pipeline.rb index fb6f6e9077d..1da0f72996b 100644 --- a/lib/banzai/pipeline/plain_markdown_pipeline.rb +++ b/lib/banzai/pipeline/plain_markdown_pipeline.rb @@ -5,7 +5,6 @@ module Banzai class PlainMarkdownPipeline < BasePipeline def self.filters FilterArray[ - Filter::PathologicalMarkdownFilter, Filter::MarkdownPreEscapeFilter, Filter::MarkdownFilter, Filter::MarkdownPostEscapeFilter diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index da2262cdf83..f8035698b9b 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -5,6 +5,7 @@ module Banzai class PostProcessPipeline < BasePipeline def self.filters @filters ||= FilterArray[ + Filter::TruncateVisibleFilter, *internal_link_filters, Filter::AbsoluteLinkFilter, Filter::BroadcastMessagePlaceholdersFilter diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index 92894575ec2..887f13b0358 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -8,6 +8,11 @@ module Bitbucket @connection = Connection.new(options) end + def last_issue(repo) + parsed_response = connection.get("/repositories/#{repo}/issues?pagelen=1&sort=-created_on&state=ALL") + Bitbucket::Representation::Issue.new(parsed_response['values'].first) + end + def issues(repo) path = "/repositories/#{repo}/issues" get_collection(path, :issue) diff --git a/lib/bulk_imports/common/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb index 915dcf1b455..5066f622d57 100644 --- a/lib/bulk_imports/common/pipelines/entity_finisher.rb +++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb @@ -24,11 +24,12 @@ module BulkImports end logger.info( - bulk_import_id: context.bulk_import.id, + bulk_import_id: context.bulk_import_id, bulk_import_entity_id: context.entity.id, bulk_import_entity_type: context.entity.source_type, pipeline_class: self.class.name, - message: "Entity #{entity.status_name}" + message: "Entity #{entity.status_name}", + importer: 'gitlab_migration' ) context.portable.try(:after_import) diff --git a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb index 0f92c1f1210..fea550b9f9d 100644 --- a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb @@ -44,6 +44,11 @@ module BulkImports wikis = client.get(context.entity.wikis_url_path).parsed_response wikis.any? + rescue BulkImports::NetworkError => e + # 403 is returned when wiki is disabled in settings + return if e.response&.forbidden? || e.response&.not_found? + + raise end def client diff --git a/lib/bulk_imports/common/rest/get_badges_query.rb b/lib/bulk_imports/common/rest/get_badges_query.rb index 60b2ebcc552..6e7cb7504ba 100644 --- a/lib/bulk_imports/common/rest/get_badges_query.rb +++ b/lib/bulk_imports/common/rest/get_badges_query.rb @@ -7,11 +7,8 @@ module BulkImports extend self def to_h(context) - resource = context.entity.pluralized_name - encoded_full_path = ERB::Util.url_encode(context.entity.source_full_path) - { - resource: [resource, encoded_full_path, 'badges'].join('/'), + resource: [context.entity.base_resource_path, 'badges'].join('/'), query: { page: context.tracker.next_page } diff --git a/lib/bulk_imports/features.rb b/lib/bulk_imports/features.rb new file mode 100644 index 00000000000..952e8e62d71 --- /dev/null +++ b/lib/bulk_imports/features.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module BulkImports + module Features + def self.enabled? + ::Feature.enabled?(:bulk_import) + end + + def self.project_migration_enabled?(destination_namespace = nil) + if destination_namespace.present? + root_ancestor = Namespace.find_by_full_path(destination_namespace)&.root_ancestor + + ::Feature.enabled?(:bulk_import_projects, root_ancestor) + else + ::Feature.enabled?(:bulk_import_projects) + end + end + end +end diff --git a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb index 1140beef48c..029b6f0f729 100644 --- a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb +++ b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb @@ -5,10 +5,8 @@ module BulkImports module Extractors class SubgroupsExtractor def extract(context) - encoded_parent_path = ERB::Util.url_encode(context.entity.source_full_path) - response = http_client(context.configuration) - .each_page(:get, "groups/#{encoded_parent_path}/subgroups") + .each_page(:get, "#{context.entity.base_resource_path}/subgroups") .flat_map(&:itself) BulkImports::Pipeline::ExtractedData.new(data: response) diff --git a/lib/bulk_imports/groups/graphql/get_group_query.rb b/lib/bulk_imports/groups/graphql/get_group_query.rb index 911b2b67d8c..0e73a7fb4b9 100644 --- a/lib/bulk_imports/groups/graphql/get_group_query.rb +++ b/lib/bulk_imports/groups/graphql/get_group_query.rb @@ -14,9 +14,9 @@ module BulkImports <<-'GRAPHQL' query($full_path: ID!) { group(fullPath: $full_path) { + id name path - full_path: fullPath description visibility emails_disabled: emailsDisabled diff --git a/lib/bulk_imports/groups/graphql/get_projects_query.rb b/lib/bulk_imports/groups/graphql/get_projects_query.rb index 3f74bbb8cce..4784baf225c 100644 --- a/lib/bulk_imports/groups/graphql/get_projects_query.rb +++ b/lib/bulk_imports/groups/graphql/get_projects_query.rb @@ -20,6 +20,7 @@ module BulkImports has_next_page: hasNextPage } nodes { + id name full_path: fullPath } diff --git a/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb index c318675e649..026b2e55713 100644 --- a/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb @@ -15,7 +15,8 @@ module BulkImports source_full_path: data['full_path'], destination_name: data['name'], destination_namespace: context.entity.group.full_path, - parent_id: context.entity.id + parent_id: context.entity.id, + source_xid: GlobalID.parse(data['id']).model_id } end diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb index 0378a9c605d..6928ce43191 100644 --- a/lib/bulk_imports/groups/stage.rb +++ b/lib/bulk_imports/groups/stage.rb @@ -90,13 +90,7 @@ module BulkImports def feature_flag_enabled? destination_namespace = @bulk_import_entity.destination_namespace - if destination_namespace.present? - root_ancestor = Namespace.find_by_full_path(destination_namespace)&.root_ancestor - - ::Feature.enabled?(:bulk_import_projects, root_ancestor) - else - ::Feature.enabled?(:bulk_import_projects) - end + BulkImports::Features.project_migration_enabled?(destination_namespace) end end end diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb index d9efcdb1ba5..83b442458dc 100644 --- a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb +++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb @@ -4,6 +4,7 @@ module BulkImports module Groups module Transformers class GroupAttributesTransformer + # rubocop: disable Style/IfUnlessModifier def transform(context, data) import_entity = context.entity @@ -11,74 +12,64 @@ module BulkImports namespace = Namespace.find_by_full_path(import_entity.destination_namespace) end - data - .then { |data| transform_name(import_entity, namespace, data) } - .then { |data| transform_path(import_entity, data) } - .then { |data| transform_full_path(data) } - .then { |data| transform_parent(context, import_entity, namespace, data) } - .then { |data| transform_visibility_level(data) } - .then { |data| transform_project_creation_level(data) } - .then { |data| transform_subgroup_creation_level(data) } - end - - private + params = { + 'name' => group_name(namespace, data), + 'path' => import_entity.destination_slug.parameterize, + 'description' => data['description'], + 'lfs_enabled' => data['lfs_enabled'], + 'emails_disabled' => data['emails_disabled'], + 'mentions_disabled' => data['mentions_disabled'], + 'share_with_group_lock' => data['share_with_group_lock'] + } - def transform_name(import_entity, namespace, data) if namespace.present? - namespace_children_names = namespace.children.pluck(:name) # rubocop: disable CodeReuse/ActiveRecord - - if namespace_children_names.include?(data['name']) - data['name'] = Uniquify.new(1).string(-> (counter) { "#{data['name']}(#{counter})" }) do |base| - namespace_children_names.include?(base) - end - end + params['parent_id'] = namespace.id end - data - end - - def transform_path(import_entity, data) - data['path'] = import_entity.destination_slug.parameterize - data - end + if data.has_key?('two_factor_grace_period') + params['two_factor_grace_period'] = data['two_factor_grace_period'] + end - def transform_full_path(data) - data.delete('full_path') - data - end + if data.has_key?('request_access_enabled') + params['request_access_enabled'] = data['request_access_enabled'] + end - def transform_parent(context, import_entity, namespace, data) - data['parent_id'] = namespace.id if namespace.present? + if data.has_key?('require_two_factor_authentication') + params['require_two_factor_authentication'] = data['require_two_factor_authentication'] + end - data - end + if data.has_key?('project_creation_level') + params['project_creation_level'] = + Gitlab::Access.project_creation_string_options[data['project_creation_level']] + end - def transform_visibility_level(data) - visibility = data['visibility'] + if data.has_key?('subgroup_creation_level') + params['subgroup_creation_level'] = + Gitlab::Access.subgroup_creation_string_options[data['subgroup_creation_level']] + end - return data unless visibility.present? + if data.has_key?('visibility') + params['visibility_level'] = Gitlab::VisibilityLevel.string_options[data['visibility']] + end - data['visibility_level'] = Gitlab::VisibilityLevel.string_options[visibility] - data.delete('visibility') - data + params end + # rubocop: enable Style/IfUnlessModifier - def transform_project_creation_level(data) - project_creation_level = data['project_creation_level'] - - return data unless project_creation_level.present? - - data['project_creation_level'] = Gitlab::Access.project_creation_string_options[project_creation_level] - data - end + private - def transform_subgroup_creation_level(data) - subgroup_creation_level = data['subgroup_creation_level'] + def group_name(namespace, data) + if namespace.present? + namespace_children_names = namespace.children.pluck(:name) # rubocop: disable CodeReuse/ActiveRecord - return data unless subgroup_creation_level.present? + if namespace_children_names.include?(data['name']) + data['name'] = Uniquify.new(1).string(-> (counter) { "#{data['name']}(#{counter})" }) do |base| + namespace_children_names.include?(base) + end + end + end - data['subgroup_creation_level'] = Gitlab::Access.subgroup_creation_string_options[subgroup_creation_level] - data + data['name'] end end end diff --git a/lib/bulk_imports/network_error.rb b/lib/bulk_imports/network_error.rb index 3514291a75d..fda4bb74a30 100644 --- a/lib/bulk_imports/network_error.rb +++ b/lib/bulk_imports/network_error.rb @@ -2,15 +2,21 @@ module BulkImports class NetworkError < Error - COUNTER_KEY = 'bulk_imports/%{entity_id}/%{stage}/%{tracker_id}/network_error/%{error}' + TRACKER_COUNTER_KEY = 'bulk_imports/%{entity_id}/%{stage}/%{tracker_id}/network_error/%{error}' + ENTITY_COUNTER_KEY = 'bulk_imports/%{entity_id}/network_error/%{error}' - RETRIABLE_EXCEPTIONS = Gitlab::HTTP::HTTP_TIMEOUT_ERRORS + RETRIABLE_EXCEPTIONS = Gitlab::HTTP::HTTP_TIMEOUT_ERRORS + [ + EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, + Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH + ].freeze RETRIABLE_HTTP_CODES = [429].freeze DEFAULT_RETRY_DELAY_SECONDS = 30 MAX_RETRIABLE_COUNT = 10 + attr_reader :response + def initialize(message = nil, response: nil) raise ArgumentError, 'message or response required' if message.blank? && response.blank? @@ -19,9 +25,9 @@ module BulkImports @response = response end - def retriable?(tracker) + def retriable?(object) if retriable_exception? || retriable_http_code? - increment(tracker) <= MAX_RETRIABLE_COUNT + increment(object) <= MAX_RETRIABLE_COUNT else false end @@ -37,8 +43,6 @@ module BulkImports private - attr_reader :response - def retriable_exception? RETRIABLE_EXCEPTIONS.include?(cause&.class) end @@ -47,15 +51,27 @@ module BulkImports RETRIABLE_HTTP_CODES.include?(response&.code) end - def increment(tracker) - key = COUNTER_KEY % { + def increment(object) + key = object.is_a?(BulkImports::Entity) ? entity_cache_key(object) : tracker_cache_key(object) + + Gitlab::Cache::Import::Caching.increment(key) + end + + def tracker_cache_key(tracker) + TRACKER_COUNTER_KEY % { stage: tracker.stage, tracker_id: tracker.id, entity_id: tracker.entity.id, error: cause.class.name } + end - Gitlab::Cache::Import::Caching.increment(key) + def entity_cache_key(entity) + ENTITY_COUNTER_KEY % { + import_id: entity.bulk_import_id, + entity_id: entity.id, + error: cause.class.name + } end end end diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb index 68b86c68619..54d5d3209f8 100644 --- a/lib/bulk_imports/pipeline.rb +++ b/lib/bulk_imports/pipeline.rb @@ -183,7 +183,11 @@ module BulkImports end def relation - class_attributes[:relation_name] + class_attributes[:relation_name] || default_relation + end + + def default_relation + self.name.demodulize.chomp('Pipeline').underscore end private diff --git a/lib/bulk_imports/pipeline/context.rb b/lib/bulk_imports/pipeline/context.rb index d753f888671..fde24cf3646 100644 --- a/lib/bulk_imports/pipeline/context.rb +++ b/lib/bulk_imports/pipeline/context.rb @@ -32,6 +32,10 @@ module BulkImports @bulk_import ||= entity.bulk_import end + def bulk_import_id + @bulk_import_id ||= bulk_import.id + end + def current_user @current_user ||= bulk_import.user end diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index c03da7d8d01..ef9575d1e96 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -54,7 +54,8 @@ module BulkImports skip!( 'Skipping pipeline due to failed entity', pipeline_step: step, - step_class: class_name + step_class: class_name, + importer: 'gitlab_migration' ) rescue BulkImports::NetworkError => e if e.retriable?(context.tracker) @@ -108,9 +109,12 @@ module BulkImports } error( + bulk_import_id: context.bulk_import_id, pipeline_step: step, exception_class: exception.class.to_s, - exception_message: exception.message + exception_message: exception.message, + message: "Pipeline failed", + importer: 'gitlab_migration' ) BulkImports::Failure.create(attributes) @@ -130,11 +134,12 @@ module BulkImports def log_params(extra) defaults = { - bulk_import_id: context.bulk_import.id, + bulk_import_id: context.bulk_import_id, bulk_import_entity_id: context.entity.id, bulk_import_entity_type: context.entity.source_type, pipeline_class: pipeline, - context_extra: context.extra + context_extra: context.extra, + importer: 'gitlab_migration' } defaults diff --git a/lib/bulk_imports/projects/graphql/get_project_query.rb b/lib/bulk_imports/projects/graphql/get_project_query.rb index 76475893ac1..a2d7094d570 100644 --- a/lib/bulk_imports/projects/graphql/get_project_query.rb +++ b/lib/bulk_imports/projects/graphql/get_project_query.rb @@ -10,6 +10,7 @@ module BulkImports <<-'GRAPHQL' query($full_path: ID!) { project(fullPath: $full_path) { + id visibility created_at: createdAt } diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 498bc11b168..723935f8aaf 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -10,6 +10,8 @@ module ContainerRegistry REGISTRY_FEATURES_HEADER = 'gitlab-container-registry-features' REGISTRY_TAG_DELETE_FEATURE = 'tag_delete' + DEFAULT_TAGS_PAGE_SIZE = 10000 + ALLOWED_REDIRECT_SCHEMES = %w[http https].freeze REDIRECT_OPTIONS = { clear_authorization_header: true, @@ -52,8 +54,11 @@ module ContainerRegistry } end - def repository_tags(name) - response_body faraday.get("/v2/#{name}/tags/list") + def repository_tags(name, page_size: DEFAULT_TAGS_PAGE_SIZE) + response = faraday.get("/v2/#{name}/tags/list") do |req| + req.params['n'] = page_size + end + response_body(response) end def repository_manifest(name, reference) diff --git a/lib/container_registry/gitlab_api_client.rb b/lib/container_registry/gitlab_api_client.rb index 2947dcb4b40..5dddd421223 100644 --- a/lib/container_registry/gitlab_api_client.rb +++ b/lib/container_registry/gitlab_api_client.rb @@ -23,6 +23,8 @@ module ContainerRegistry MAX_TAGS_PAGE_SIZE = 1000 + UnsuccessfulResponseError = Class.new(StandardError) + def self.supports_gitlab_api? with_dummy_client(return_value_if_disabled: false) do |client| client.supports_gitlab_api? @@ -105,12 +107,22 @@ module ContainerRegistry def tags(path, page_size: 100, last: nil) limited_page_size = [page_size, MAX_TAGS_PAGE_SIZE].min with_token_faraday do |faraday_client| - response = faraday_client.get("/gitlab/v1/repositories/#{path}/tags/list/") do |req| + url = "/gitlab/v1/repositories/#{path}/tags/list/" + response = faraday_client.get(url) do |req| req.params['n'] = limited_page_size req.params['last'] = last if last end - break {} unless response.success? + unless response.success? + Gitlab::ErrorTracking.log_exception( + UnsuccessfulResponseError.new, + class: self.class.name, + url: url, + status_code: response.status + ) + + break {} + end link_parser = Gitlab::Utils::LinkHeaderParser.new(response.headers['link']) diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb index edfc39aea0c..6af24451322 100644 --- a/lib/feature/shared.rb +++ b/lib/feature/shared.rb @@ -53,8 +53,6 @@ module Feature default_enabled: false, example: <<-EOS experiment(:my_experiment, project: project, actor: current_user) { ...variant code... } - # or - Gitlab::Experimentation.in_experiment_group?(:my_experiment, subject: current_user) EOS } }.freeze diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 0f0ecd82a32..1920e1443da 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -21,6 +21,7 @@ module Gitlab :related_class, :feature_category, :artifact_size, + :artifact_used_cdn, :artifacts_dependencies_size, :artifacts_dependencies_count, :root_caller_id @@ -38,6 +39,7 @@ module Gitlab Attribute.new(:related_class, String), Attribute.new(:feature_category, String), Attribute.new(:artifact, ::Ci::JobArtifact), + Attribute.new(:artifact_used_cdn, Object), Attribute.new(:artifacts_dependencies_size, Integer), Attribute.new(:artifacts_dependencies_count, Integer), Attribute.new(:root_caller_id, String) @@ -91,6 +93,7 @@ module Gitlab assign_hash_if_value(hash, :remote_ip) assign_hash_if_value(hash, :related_class) assign_hash_if_value(hash, :feature_category) + assign_hash_if_value(hash, :artifact_used_cdn) assign_hash_if_value(hash, :artifacts_dependencies_size) assign_hash_if_value(hash, :artifacts_dependencies_count) diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 6213dd203c4..c567df8e133 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -221,6 +221,8 @@ module Gitlab end if token.user.can_log_in_with_non_expired_password? || token.user.project_bot? + ::PersonalAccessTokens::LastUsedService.new(token).execute + Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) end end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 1fed2b263da..26be7c8aa60 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -217,11 +217,7 @@ module Gitlab def build_new_user(skip_confirmation: true) user_params = user_attributes.merge(skip_confirmation: skip_confirmation) - new_user = Users::AuthorizedBuildService.new(nil, user_params).execute - - persist_accepted_terms_if_required(new_user) - - new_user + Users::AuthorizedBuildService.new(nil, user_params).execute end def user_attributes @@ -249,15 +245,6 @@ module Gitlab } end - def persist_accepted_terms_if_required(new_user) - if Feature.enabled?(:update_oauth_registration_flow) && - Gitlab::CurrentSettings.current_application_settings.enforce_terms? - - terms = ApplicationSetting::Term.latest - Users::RespondToTermsService.new(new_user, terms).execute(accepted: true) - end - end - def sync_profile_from_provider? Gitlab::Auth::OAuth::Provider.sync_profile_from_provider?(auth_hash.provider) end diff --git a/lib/gitlab/auth_logger.rb b/lib/gitlab/auth_logger.rb index 6d3edba02b0..763430df335 100644 --- a/lib/gitlab/auth_logger.rb +++ b/lib/gitlab/auth_logger.rb @@ -7,3 +7,5 @@ module Gitlab end end end + +Gitlab::AuthLogger.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_epic_cache_counts.rb b/lib/gitlab/background_migration/backfill_epic_cache_counts.rb new file mode 100644 index 00000000000..bd61d1a0f07 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_epic_cache_counts.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class BackfillEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob + def perform; end + end + # rubocop: enable Style/Documentation + end +end + +# rubocop: disable Layout/LineLength +Gitlab::BackgroundMigration::BackfillEpicCacheCounts.prepend_mod_with('Gitlab::BackgroundMigration::BackfillEpicCacheCounts') +# rubocop: enable Layout/LineLength diff --git a/lib/gitlab/background_migration/backfill_internal_on_notes.rb b/lib/gitlab/background_migration/backfill_internal_on_notes.rb new file mode 100644 index 00000000000..300f2cff6ca --- /dev/null +++ b/lib/gitlab/background_migration/backfill_internal_on_notes.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This syncs the data to `internal` from `confidential` as we rename the column. + class BackfillInternalOnNotes < BatchedMigrationJob + scope_to -> (relation) { relation.where(confidential: true) } + + def perform + each_sub_batch(operation_name: :update_all) do |sub_batch| + sub_batch.update_all(internal: true) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_namespace_details.rb b/lib/gitlab/background_migration/backfill_namespace_details.rb new file mode 100644 index 00000000000..b8a51b576b6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_details.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill namespace_details for a range of namespaces + class BackfillNamespaceDetails < ::Gitlab::BackgroundMigration::BatchedMigrationJob + def perform + each_sub_batch(operation_name: :backfill_namespace_details) do |sub_batch| + upsert_namespace_details(sub_batch) + end + end + + def upsert_namespace_details(relation) + connection.execute( + <<~SQL + INSERT INTO namespace_details (description, description_html, cached_markdown_version, created_at, updated_at, namespace_id) + SELECT namespaces.description, namespaces.description_html, namespaces.cached_markdown_version, now(), now(), namespaces.id + FROM namespaces + WHERE namespaces.id IN(#{relation.select(:id).to_sql}) + AND namespaces.type <> 'Project' + ON CONFLICT (namespace_id) DO NOTHING; + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb b/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb new file mode 100644 index 00000000000..c3e1019b72f --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for deleting orphaned operational vulnerabilities (without findings) + class DeleteOrphanedOperationalVulnerabilities < ::Gitlab::BackgroundMigration::BatchedMigrationJob + REPORT_TYPES = { + cluster_image_scanning: 7, + custom: 99 + }.freeze + + NOT_EXISTS_SQL = <<-SQL + NOT EXISTS ( + SELECT FROM vulnerability_occurrences + WHERE "vulnerability_occurrences"."vulnerability_id" = "vulnerabilities"."id" + ) + SQL + + scope_to ->(relation) do + relation + .where(report_type: [REPORT_TYPES[:cluster_image_scanning], REPORT_TYPES[:custom]]) + end + + def perform + each_sub_batch(operation_name: :delete_orphaned_operational_vulnerabilities) do |sub_batch| + sub_batch + .where(NOT_EXISTS_SQL) + .delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/destroy_invalid_members.rb b/lib/gitlab/background_migration/destroy_invalid_members.rb new file mode 100644 index 00000000000..7d78795bea9 --- /dev/null +++ b/lib/gitlab/background_migration/destroy_invalid_members.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class DestroyInvalidMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation + scope_to ->(relation) { relation.where(member_namespace_id: nil) } + + def perform + each_sub_batch(operation_name: :delete_all) do |sub_batch| + deleted_members_data = sub_batch.map do |m| + { id: m.id, source_id: m.source_id, source_type: m.source_type } + end + + deleted_count = sub_batch.delete_all + + Gitlab::AppLogger.info({ message: 'Removing invalid member records', + deleted_count: deleted_count, + deleted_member_data: deleted_members_data }) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb b/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb new file mode 100644 index 00000000000..2257dc016be --- /dev/null +++ b/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class doesn't delete merge request level rules + # as this feature exists only in EE + class PopulateApprovalMergeRequestRulesWithSecurityOrchestration < BatchedMigrationJob + def perform; end + end + end +end + +# rubocop:disable Layout/LineLength +Gitlab::BackgroundMigration::PopulateApprovalMergeRequestRulesWithSecurityOrchestration.prepend_mod_with('Gitlab::BackgroundMigration::PopulateApprovalMergeRequestRulesWithSecurityOrchestration') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb b/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb new file mode 100644 index 00000000000..1d0c0010551 --- /dev/null +++ b/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class doesn't delete merge request level rules + # as this feature exists only in EE + class PopulateApprovalProjectRulesWithSecurityOrchestration < BatchedMigrationJob + def perform; end + end + end +end + +# rubocop:disable Layout/LineLength +Gitlab::BackgroundMigration::PopulateApprovalProjectRulesWithSecurityOrchestration.prepend_mod_with('Gitlab::BackgroundMigration::PopulateApprovalProjectRulesWithSecurityOrchestration') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb new file mode 100644 index 00000000000..952f3b0e3c3 --- /dev/null +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify duplicate token_encrypted values in ci_runners table in batches + class ResetDuplicateCiRunnersTokenEncryptedValues < BatchedMigrationJob + def perform + each_sub_batch(operation_name: :nullify_duplicate_ci_runner_token_encrypted_values) do |sub_batch| + # Reset duplicate runner encrypted tokens that would prevent creating an unique index. + nullify_duplicate_ci_runner_token_encrypted_values(sub_batch) + end + end + + private + + def nullify_duplicate_ci_runner_token_encrypted_values(sub_batch) + batchable_model = define_batchable_model(batch_table, connection: connection) + + duplicate_tokens = batchable_model + .where(token_encrypted: sub_batch.select(:token_encrypted).distinct) + .group(:token_encrypted) + .having('COUNT(*) > 1') + .pluck(:token_encrypted) + + return if duplicate_tokens.empty? + + batchable_model.where(token_encrypted: duplicate_tokens).update_all(token_encrypted: nil) + end + end + end +end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb new file mode 100644 index 00000000000..cfd6a4e4091 --- /dev/null +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify duplicate token values in ci_runners table in batches + class ResetDuplicateCiRunnersTokenValues < BatchedMigrationJob + def perform + each_sub_batch(operation_name: :nullify_duplicate_ci_runner_token_values) do |sub_batch| + # Reset duplicate runner tokens that would prevent creating an unique index. + nullify_duplicate_ci_runner_token_values(sub_batch) + end + end + + private + + def nullify_duplicate_ci_runner_token_values(sub_batch) + batchable_model = define_batchable_model(batch_table, connection: connection) + + duplicate_tokens = batchable_model + .where(token: sub_batch.select(:token).distinct) + .group(:token) + .having('COUNT(*) > 1') + .pluck(:token) + + batchable_model.where(token: duplicate_tokens).update_all(token: nil) if duplicate_tokens.any? + end + end + end +end diff --git a/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb b/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb new file mode 100644 index 00000000000..84183753158 --- /dev/null +++ b/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The `ci_pipeline_artifacts.locked` column was added in + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/97194 to + # speed up the finding of expired, pipeline artifacts. By default, + # the value is "unknown" (2), but the correct value should be the + # value of the associated `ci_pipelines.locked` value. This class + # does an UPDATE join to make the values match. + class UpdateCiPipelineArtifactsUnknownLockedStatus < BatchedMigrationJob + def perform + connection.exec_query(<<~SQL) + UPDATE ci_pipeline_artifacts + SET locked = ci_pipelines.locked + FROM ci_pipelines + WHERE ci_pipeline_artifacts.id BETWEEN #{start_id} AND #{end_id} + AND ci_pipeline_artifacts.locked = 2 + AND ci_pipelines.id = ci_pipeline_artifacts.pipeline_id; + SQL + end + end + end +end diff --git a/lib/gitlab/batch_pop_queueing.rb b/lib/gitlab/batch_pop_queueing.rb deleted file mode 100644 index 103ce644f2b..00000000000 --- a/lib/gitlab/batch_pop_queueing.rb +++ /dev/null @@ -1,113 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - ## - # This class is a queuing system for processing expensive tasks in an atomic manner - # with batch poping to let you optimize the total processing time. - # - # In usual queuing system, the first item started being processed immediately - # and the following items wait until the next items have been popped from the queue. - # On the other hand, this queueing system, the former part is same, however, - # it pops the enqueued items as batch. This is especially useful when you want to - # drop redundant items from the queue in order to process important items only, - # thus it's more efficient than the traditional queueing system. - # - # Caveats: - # - The order of the items are not guaranteed because of `sadd` (Redis Sets). - # - # Example: - # ``` - # class TheWorker - # def perform - # result = Gitlab::BatchPopQueueing.new('feature', 'queue').safe_execute([item]) do |items_in_queue| - # item = extract_the_most_important_item_from(items_in_queue) - # expensive_process(item) - # end - # - # if result[:status] == :finished && result[:new_items].present? - # item = extract_the_most_important_item_from(items_in_queue) - # TheWorker.perform_async(item.id) - # end - # end - # end - # ``` - # - class BatchPopQueueing - attr_reader :namespace, :queue_id - - EXTRA_QUEUE_EXPIRE_WINDOW = 1.hour - MAX_COUNTS_OF_POP_ALL = 1000 - - # Initialize queue - # - # @param [String] namespace The namespace of the exclusive lock and queue key. Typically, it's a feature name. - # @param [String] queue_id The identifier of the queue. - # @return [Boolean] - def initialize(namespace, queue_id) - raise ArgumentError if namespace.empty? || queue_id.empty? - - @namespace = namespace - @queue_id = queue_id - end - - ## - # Execute the given block in an exclusive lock. - # If there is the other thread has already working on the block, - # it enqueues the items without processing the block. - # - # @param [Array<String>] new_items New items to be added to the queue. - # @param [Time] lock_timeout The timeout of the exclusive lock. Generally, this value should be longer than the maximum prosess timing of the given block. - # @return [Hash] - # - status => One of the `:enqueued` or `:finished`. - # - new_items => Newly enqueued items during the given block had been processed. - # - # NOTE: If an exception is raised in the block, the poppped items will not be recovered. - # We should NOT re-enqueue the items in this case because it could end up in an infinite loop. - def safe_execute(new_items, lock_timeout: 10.minutes, &block) - enqueue(new_items, lock_timeout + EXTRA_QUEUE_EXPIRE_WINDOW) - - lease = Gitlab::ExclusiveLease.new(lock_key, timeout: lock_timeout) - - return { status: :enqueued } unless uuid = lease.try_obtain - - begin - all_args = pop_all - - yield all_args if block - - { status: :finished, new_items: peek_all } - ensure - Gitlab::ExclusiveLease.cancel(lock_key, uuid) - end - end - - private - - def lock_key - @lock_key ||= "batch_pop_queueing:lock:#{namespace}:#{queue_id}" - end - - def queue_key - @queue_key ||= "batch_pop_queueing:queue:#{namespace}:#{queue_id}" - end - - def enqueue(items, expire_time) - Gitlab::Redis::Queues.with do |redis| - redis.sadd(queue_key, items) - redis.expire(queue_key, expire_time.to_i) - end - end - - def pop_all - Gitlab::Redis::Queues.with do |redis| - redis.spop(queue_key, MAX_COUNTS_OF_POP_ALL) - end - end - - def peek_all - Gitlab::Redis::Queues.with do |redis| - redis.smembers(queue_key) - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index d58de7eb211..7de6be45349 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -67,6 +67,14 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + def allocate_issues_internal_id!(project, client) + last_bitbucket_issue = client.last_issue(repo) + + return unless last_bitbucket_issue + + Issue.track_project_iid!(project, last_bitbucket_issue.iid) + end + def repo @repo ||= client.repo(project.import_source) end @@ -84,6 +92,10 @@ module Gitlab def import_issues return unless repo.issues_enabled? + # If a user creates an issue while the import is in progress, this can lead to an import failure. + # The workaround is to allocate IIDs before starting the importer. + allocate_issues_internal_id!(project, client) + create_labels issue_type_id = WorkItems::Type.default_issue_type.id diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 99ce1119c17..9209c9b4927 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -108,7 +108,7 @@ module Gitlab return self.loaded unless self.loaded.nil? Gitlab::Redis::Cache.with do |redis| - redis.exists(cache_key) + redis.exists?(cache_key) # rubocop:disable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/cache/helpers.rb b/lib/gitlab/cache/helpers.rb index 48b6ca59367..024fa48c066 100644 --- a/lib/gitlab/cache/helpers.rb +++ b/lib/gitlab/cache/helpers.rb @@ -126,7 +126,6 @@ module Gitlab end def increment_cache_metric(render_type:, total_count:, miss_count:) - return unless Feature.enabled?(:add_timing_to_certain_cache_actions) return unless caller_id metric_name = :cached_object_operations_total @@ -146,17 +145,13 @@ module Gitlab end def time_action(render_type:, &block) - if Feature.enabled?(:add_timing_to_certain_cache_actions) - real_start = Gitlab::Metrics::System.monotonic_time + real_start = Gitlab::Metrics::System.monotonic_time - presented_object = yield + presented_object = yield - real_duration_histogram(render_type).observe({}, Gitlab::Metrics::System.monotonic_time - real_start) + real_duration_histogram(render_type).observe({}, Gitlab::Metrics::System.monotonic_time - real_start) - presented_object - else - yield - end + presented_object end def real_duration_histogram(render_type) diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index 1034f5eacef..4069a683ceb 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -41,7 +41,6 @@ module Gitlab def find_modified_paths(pipeline) return unless pipeline - return pipeline.modified_paths unless ::Feature.enabled?(:ci_rules_changes_compare, pipeline.project) compare_to_sha = find_compare_to_sha(pipeline) diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 438fa1cb3b2..661c6fb87e3 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -85,6 +85,10 @@ module Gitlab root.workflow_entry.rules_value end + def workflow_name + root.workflow_entry.name + end + def normalized_jobs @normalized_jobs ||= Ci::Config::Normalizer.new(jobs).normalize_jobs end diff --git a/lib/gitlab/ci/config/entry/current_variables.rb b/lib/gitlab/ci/config/entry/current_variables.rb deleted file mode 100644 index 3b6721ec92d..00000000000 --- a/lib/gitlab/ci/config/entry/current_variables.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - ## - # Entry that represents CI/CD variables. - # The class will be renamed to `Variables` when removing the FF `ci_variables_refactoring_to_variable`. - # - class CurrentVariables < ::Gitlab::Config::Entry::ComposableHash - include ::Gitlab::Config::Entry::Validatable - - validations do - validates :config, type: Hash - end - - # Enable these lines when removing the FF `ci_variables_refactoring_to_variable` - # and renaming this class to `Variables`. - # def self.default(**) - # {} - # end - - def value - @entries.to_h do |key, entry| - [key.to_s, entry.value] - end - end - - def value_with_data - @entries.to_h do |key, entry| - [key.to_s, entry.value_with_data] - end - end - - private - - def composable_class(_name, _config) - Entry::Variable - end - - def composable_metadata - { allowed_value_data: opt(:allowed_value_data) } - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/legacy_variables.rb b/lib/gitlab/ci/config/entry/legacy_variables.rb deleted file mode 100644 index 5379f707537..00000000000 --- a/lib/gitlab/ci/config/entry/legacy_variables.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - ## - # Entry that represents environment variables. - # This is legacy implementation and will be removed with the FF `ci_variables_refactoring_to_variable`. - # - class LegacyVariables < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable - - ALLOWED_VALUE_DATA = %i[value description].freeze - - validations do - validates :config, variables: { allowed_value_data: ALLOWED_VALUE_DATA }, if: :use_value_data? - validates :config, variables: true, unless: :use_value_data? - end - - def value - @config.to_h { |key, value| [key.to_s, expand_value(value)[:value]] } - end - - def value_with_data - @config.to_h { |key, value| [key.to_s, expand_value(value)] } - end - - def use_value_data? - opt(:use_value_data) - end - - private - - def expand_value(value) - if value.is_a?(Hash) - { value: value[:value].to_s, description: value[:description] }.compact - else - { value: value.to_s } - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 57e89bd7bc5..1d7d8617c74 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -48,10 +48,9 @@ module Gitlab description: 'Script that will be executed after each job.', reserved: true - # use_value_data will be removed with the FF ci_variables_refactoring_to_variable entry :variables, Entry::Variables, description: 'Environment variables that will be used.', - metadata: { use_value_data: true, allowed_value_data: %i[value description] }, + metadata: { allowed_value_data: %i[value description], allow_array_value: true }, reserved: true entry :stages, Entry::Stages, diff --git a/lib/gitlab/ci/config/entry/timeout.rb b/lib/gitlab/ci/config/entry/timeout.rb index 0bffa9340de..5769ea22b06 100644 --- a/lib/gitlab/ci/config/entry/timeout.rb +++ b/lib/gitlab/ci/config/entry/timeout.rb @@ -5,7 +5,7 @@ module Gitlab class Config module Entry ## - # Entry that represents the interrutible value. + # Entry that represents the interruptible value. # class Timeout < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable diff --git a/lib/gitlab/ci/config/entry/variable.rb b/lib/gitlab/ci/config/entry/variable.rb index 253888aadeb..54c153c8b07 100644 --- a/lib/gitlab/ci/config/entry/variable.rb +++ b/lib/gitlab/ci/config/entry/variable.rb @@ -10,6 +10,7 @@ module Gitlab class Variable < ::Gitlab::Config::Entry::Simplifiable strategy :SimpleVariable, if: -> (config) { SimpleVariable.applies_to?(config) } strategy :ComplexVariable, if: -> (config) { ComplexVariable.applies_to?(config) } + strategy :ComplexArrayVariable, if: -> (config) { ComplexArrayVariable.applies_to?(config) } class SimpleVariable < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable @@ -39,7 +40,7 @@ module Gitlab class << self def applies_to?(config) - config.is_a?(Hash) + config.is_a?(Hash) && !config[:value].is_a?(Array) end end @@ -86,6 +87,34 @@ module Gitlab end end + class ComplexArrayVariable < ComplexVariable + include ::Gitlab::Config::Entry::Validatable + + class << self + def applies_to?(config) + config.is_a?(Hash) && config[:value].is_a?(Array) + end + end + + validations do + validates :config_value, array_of_strings: true, allow_nil: false, if: :config_value_defined? + + validate do + next if opt(:allow_array_value) + + errors.add(:config, 'value must be an alphanumeric string') + end + end + + def value + config_value.first + end + + def value_with_data + super.merge(value_options: config_value).compact + end + end + class UnknownStrategy < ::Gitlab::Config::Entry::Node def errors ["variable definition must be either a string or a hash"] diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index 0284958d9d4..4430a11dda7 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -6,20 +6,38 @@ module Gitlab module Entry ## # Entry that represents CI/CD variables. - # CurrentVariables will be renamed to this class when removing the FF `ci_variables_refactoring_to_variable`. - # - class Variables - def self.new(...) - if YamlProcessor::FeatureFlags.enabled?(:ci_variables_refactoring_to_variable) - CurrentVariables.new(...) - else - LegacyVariables.new(...) - end + class Variables < ::Gitlab::Config::Entry::ComposableHash + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: Hash end def self.default(**) {} end + + def value + @entries.to_h do |key, entry| + [key.to_s, entry.value] + end + end + + def value_with_data + @entries.to_h do |key, entry| + [key.to_s, entry.value_with_data] + end + end + + private + + def composable_class(_name, _config) + Entry::Variable + end + + def composable_metadata + { allowed_value_data: opt(:allowed_value_data), allow_array_value: opt(:allow_array_value) } + end end end end diff --git a/lib/gitlab/ci/config/entry/workflow.rb b/lib/gitlab/ci/config/entry/workflow.rb index 5bc992a38a0..691d9e2d48b 100644 --- a/lib/gitlab/ci/config/entry/workflow.rb +++ b/lib/gitlab/ci/config/entry/workflow.rb @@ -6,12 +6,17 @@ module Gitlab module Entry class Workflow < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[rules].freeze + ALLOWED_KEYS = %i[rules name].freeze + + attributes :name validations do validates :config, type: Hash validates :config, allowed_keys: ALLOWED_KEYS + validates :name, allow_nil: true, length: { minimum: 1, maximum: 255 } end entry :rules, Entry::Rules, diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index ec628399785..138e79db331 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -10,7 +10,6 @@ module Gitlab TimeoutError = Class.new(StandardError) MAX_INCLUDES = 100 - TRIAL_MAX_INCLUDES = 250 include ::Gitlab::Utils::StrongMemoize @@ -31,7 +30,7 @@ module Gitlab @expandset = Set.new @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) - @max_includes = Feature.enabled?(:ci_increase_includes_to_250, project) ? TRIAL_MAX_INCLUDES : MAX_INCLUDES + @max_includes = MAX_INCLUDES yield self if block_given? end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index d3e7210b820..d82ca875e76 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -12,7 +12,7 @@ module Gitlab self.new(build, ttl: build.metadata_timeout).encoded end - def initialize(build, ttl: nil) + def initialize(build, ttl:) @build = build @ttl = ttl end diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb index 4e01688a955..cfefa79d9e0 100644 --- a/lib/gitlab/ci/jwt_v2.rb +++ b/lib/gitlab/ci/jwt_v2.rb @@ -3,13 +3,27 @@ module Gitlab module Ci class JwtV2 < Jwt + DEFAULT_AUD = Settings.gitlab.base_url + + def self.for_build(build, aud: DEFAULT_AUD) + new(build, ttl: build.metadata_timeout, aud: aud).encoded + end + + def initialize(build, ttl:, aud:) + super(build, ttl: ttl) + + @aud = aud + end + private + attr_reader :aud + def reserved_claims super.merge( iss: Settings.gitlab.base_url, sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}", - aud: Settings.gitlab.base_url + aud: aud ) end end diff --git a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb index 00ca723b258..c76a4309779 100644 --- a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb +++ b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb @@ -23,8 +23,7 @@ module Gitlab ::Gitlab::Ci::Reports::Sbom::Source.new( type: :dependency_scanning, - data: data, - fingerprint: fingerprint + data: data ) end @@ -37,10 +36,6 @@ module Gitlab data.dig(*keys).present? end end - - def fingerprint - Digest::SHA256.hexdigest(data.to_json) - end end end end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index da7faaab6ff..0c117d5f214 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -44,31 +44,15 @@ module Gitlab attr_reader :json_data, :report, :validate def valid? - # We want validation to happen regardless of VALIDATE_SCHEMA - # CI variable. - # - # Previously it controlled BOTH validation and enforcement of - # schema validation result. - # - # After 15.0 we will enforce schema validation by default - # See: https://gitlab.com/groups/gitlab-org/-/epics/6968 - schema_validator.deprecation_warnings.each { |deprecation_warning| report.add_warning('Schema', deprecation_warning) } - - if validate - schema_validation_passed = schema_validator.valid? + return true unless validate - # Validation warnings are errors - schema_validator.errors.each { |error| report.add_error('Schema', error) } - schema_validator.warnings.each { |warning| report.add_error('Schema', warning) } + schema_validation_passed = schema_validator.valid? - schema_validation_passed - else - # Validation warnings are warnings - schema_validator.errors.each { |error| report.add_warning('Schema', error) } - schema_validator.warnings.each { |warning| report.add_warning('Schema', warning) } + schema_validator.errors.each { |error| report.add_error('Schema', error) } + schema_validator.deprecation_warnings.each { |deprecation_warning| report.add_warning('Schema', deprecation_warning) } + schema_validator.warnings.each { |warning| report.add_warning('Schema', warning) } - true - end + schema_validation_passed end def schema_validator @@ -216,7 +200,22 @@ module Gitlab external_id: scanner_data['id'], name: scanner_data['name'], vendor: scanner_data.dig('vendor', 'name'), - version: scanner_data.dig('version'))) + version: scanner_data.dig('version'), + primary_identifiers: create_scan_primary_identifiers)) + end + + # TODO: primary_identifiers should be initialized on the + # scan itself but we do not currently parse scans through `MergeReportsService` + def create_scan_primary_identifiers + return unless scan_data.is_a?(Hash) && scan_data.dig('primary_identifiers') + + scan_data.dig('primary_identifiers').map do |identifier| + ::Gitlab::Ci::Reports::Security::Identifier.new( + external_type: identifier['type'], + external_id: identifier['value'], + name: identifier['name'], + url: identifier['url']) + end end def create_identifiers(identifiers) diff --git a/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb b/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb deleted file mode 100644 index 24613a441be..00000000000 --- a/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Parsers - module Security - module Concerns - module DeprecatedSyntax - extend ActiveSupport::Concern - - included do - extend ::Gitlab::Utils::Override - - override :parse_report - end - - def report_data - @report_data ||= begin - data = super - - if data.is_a?(Array) - data = { - "version" => self.class::DEPRECATED_REPORT_VERSION, - "vulnerabilities" => data - } - end - - data - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/parsers/security/sast.rb b/lib/gitlab/ci/parsers/security/sast.rb index e3c62614cd8..3d999f20f1e 100644 --- a/lib/gitlab/ci/parsers/security/sast.rb +++ b/lib/gitlab/ci/parsers/security/sast.rb @@ -5,10 +5,6 @@ module Gitlab module Parsers module Security class Sast < Common - include Security::Concerns::DeprecatedSyntax - - DEPRECATED_REPORT_VERSION = "1.2" - private def create_location(location_data) diff --git a/lib/gitlab/ci/parsers/security/secret_detection.rb b/lib/gitlab/ci/parsers/security/secret_detection.rb index c6d95c1d391..175731b6b64 100644 --- a/lib/gitlab/ci/parsers/security/secret_detection.rb +++ b/lib/gitlab/ci/parsers/security/secret_detection.rb @@ -5,10 +5,6 @@ module Gitlab module Parsers module Security class SecretDetection < Common - include Security::Concerns::DeprecatedSyntax - - DEPRECATED_REPORT_VERSION = "1.2" - private def create_location(location_data) diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 28d6620e5c4..627a1f58715 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -7,14 +7,14 @@ module Gitlab module Validators class SchemaValidator SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], - container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], - coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], - dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], - api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], - dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], - sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], - secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0] + cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], + container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], + coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], + dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], + api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], + dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], + sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], + secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2] }.freeze VERSIONS_TO_REMOVE_IN_16_0 = [].freeze diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..0fcab3cd8bb --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/cluster-image-scanning-report-format.json @@ -0,0 +1,980 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.1" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/container-scanning-report-format.json new file mode 100644 index 00000000000..c08d0b45ced --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/container-scanning-report-format.json @@ -0,0 +1,914 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.1" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+(:[^:]+)?$", + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$", + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..e1ee91de23c --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/coverage-fuzzing-report-format.json @@ -0,0 +1,870 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.1" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/dast-report-format.json new file mode 100644 index 00000000000..ba2b31cf6aa --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/dast-report-format.json @@ -0,0 +1,1275 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.1" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/dependency-scanning-report-format.json new file mode 100644 index 00000000000..2bf207ce2c2 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/dependency-scanning-report-format.json @@ -0,0 +1,978 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.1" + }, + "required": [ + "dependency_files", + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/sast-report-format.json new file mode 100644 index 00000000000..c3f3bf8265f --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/sast-report-format.json @@ -0,0 +1,865 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.1" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/secret-detection-report-format.json new file mode 100644 index 00000000000..9f7c4a45466 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/secret-detection-report-format.json @@ -0,0 +1,888 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.1" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..b753a44a3d4 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/cluster-image-scanning-report-format.json @@ -0,0 +1,980 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.2" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/container-scanning-report-format.json new file mode 100644 index 00000000000..85671d03a27 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/container-scanning-report-format.json @@ -0,0 +1,912 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.2" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..33568a246fa --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/coverage-fuzzing-report-format.json @@ -0,0 +1,870 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.2" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/dast-report-format.json new file mode 100644 index 00000000000..eb141f044d5 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/dast-report-format.json @@ -0,0 +1,1275 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.2" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/dependency-scanning-report-format.json new file mode 100644 index 00000000000..31905180019 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/dependency-scanning-report-format.json @@ -0,0 +1,978 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.2" + }, + "required": [ + "dependency_files", + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/sast-report-format.json new file mode 100644 index 00000000000..efc9715aafb --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/sast-report-format.json @@ -0,0 +1,865 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.2" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/secret-detection-report-format.json new file mode 100644 index 00000000000..adbd01760d7 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/secret-detection-report-format.json @@ -0,0 +1,888 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.2" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 9c12d46cede..07a3aff1862 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -11,9 +11,11 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def perform! + ff_enabled = Feature.enabled?(:ci_skip_auto_cancelation_on_child_pipelines, project) + return if ff_enabled && pipeline.parent_pipeline? # skip if child pipeline return unless project.auto_cancel_pending_pipelines? - Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables| + Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines(ff_enabled), name: 'cancel_pending_pipelines') do |cancelables| cancelables.select(:id).each_batch(of: BATCH_SIZE) do |cancelables_batch| auto_cancel_interruptible_pipelines(cancelables_batch.ids) end @@ -27,13 +29,19 @@ module Gitlab private - def auto_cancelable_pipelines - project.all_pipelines.created_after(1.week.ago) + def auto_cancelable_pipelines(ff_enabled) + relation = project.all_pipelines + .created_after(1.week.ago) .ci_and_parent_sources .for_ref(pipeline.ref) - .id_not_in(pipeline.same_family_pipeline_ids) .where_not_sha(project.commit(pipeline.ref).try(:id)) .alive_or_scheduled + + if ff_enabled + relation.id_not_in(pipeline.id) + else + relation.id_not_in(pipeline.same_family_pipeline_ids) + end end def auto_cancel_interruptible_pipelines(pipeline_ids) @@ -41,6 +49,14 @@ module Gitlab .id_in(pipeline_ids) .with_only_interruptible_builds .each do |cancelable_pipeline| + Gitlab::AppLogger.info( + class: self.class.name, + message: "Pipeline #{pipeline.id} auto-canceling pipeline #{cancelable_pipeline.id}", + canceled_pipeline_id: cancelable_pipeline.id, + canceled_by_pipeline_id: pipeline.id, + canceled_by_pipeline_source: pipeline.source + ) + # cascade_to_children not needed because we iterate through descendants here cancelable_pipeline.cancel_running( auto_canceled_by_pipeline_id: pipeline.id, diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 14c320f77bf..76d4a05bf30 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -121,11 +121,7 @@ module Gitlab end def observe_jobs_count_in_alive_pipelines - jobs_count = if Feature.enabled?(:ci_limit_active_jobs_early, project) - project.all_pipelines.jobs_count_in_alive_pipelines - else - project.all_pipelines.builds_count_in_alive_pipelines - end + jobs_count = project.all_pipelines.jobs_count_in_alive_pipelines metrics.active_jobs_histogram .observe({ plan: project.actual_plan_name }, jobs_count) diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index a14dec48619..d41213ef6dd 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -9,15 +9,6 @@ module Gitlab include Chain::Helpers include ::Gitlab::Utils::StrongMemoize - SOURCES = [ - Gitlab::Ci::Pipeline::Chain::Config::Content::Parameter, - Gitlab::Ci::Pipeline::Chain::Config::Content::Bridge, - Gitlab::Ci::Pipeline::Chain::Config::Content::Repository, - Gitlab::Ci::Pipeline::Chain::Config::Content::ExternalProject, - Gitlab::Ci::Pipeline::Chain::Config::Content::Remote, - Gitlab::Ci::Pipeline::Chain::Config::Content::AutoDevops - ].freeze - def perform! if pipeline_config&.exists? @pipeline.build_pipeline_config(content: pipeline_config.content) @@ -36,8 +27,6 @@ module Gitlab def pipeline_config strong_memoize(:pipeline_config) do - next legacy_find_config if ::Feature.disabled?(:ci_project_pipeline_config_refactoring, project) - ::Gitlab::Ci::ProjectConfig.new( project: project, sha: @pipeline.sha, custom_content: @command.content, @@ -45,24 +34,9 @@ module Gitlab ) end end - - def legacy_find_config - sources.each do |source| - config = source.new(@pipeline, @command) - return config if config.exists? - end - - nil - end - - def sources - SOURCES - end end end end end end end - -Gitlab::Ci::Pipeline::Chain::Config::Content.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Config::Content') diff --git a/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb b/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb deleted file mode 100644 index 4947e2eb879..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class AutoDevops < Source - def content - strong_memoize(:content) do - next unless project&.auto_devops_enabled? - - template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name) - YAML.dump('include' => [{ 'template' => template.full_name }]) - end - end - - def source - :auto_devops_source - end - - private - - def template_name - 'Auto-DevOps' - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb b/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb deleted file mode 100644 index 39ffa2d4e25..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class Bridge < Source - def content - return unless @command.bridge - - @command.bridge.yaml_for_downstream - end - - def source - :bridge_source - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb b/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb deleted file mode 100644 index 092e7d43371..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class ExternalProject < Source - def content - strong_memoize(:content) do - next unless external_project_path? - - path_file, path_project, ref = extract_location_tokens - - config_location = { 'project' => path_project, 'file' => path_file } - config_location['ref'] = ref if ref.present? - - YAML.dump('include' => [config_location]) - end - end - - def source - :external_project_source - end - - private - - # Example: path/to/.gitlab-ci.yml@another-group/another-project - def external_project_path? - ci_config_path =~ /\A.+(yml|yaml)@.+\z/ - end - - # Example: path/to/.gitlab-ci.yml@another-group/another-project:refname - def extract_location_tokens - path_file, path_project = ci_config_path.split('@', 2) - - if path_project.include? ":" - project, ref = path_project.split(':', 2) - [path_file, project, ref] - else - [path_file, path_project] - end - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb deleted file mode 100644 index 9954aedc4b7..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class Parameter < Source - UnsupportedSourceError = Class.new(StandardError) - - def content - strong_memoize(:content) do - next unless command.content.present? - - command.content - end - end - - def source - :parameter_source - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb deleted file mode 100644 index 4990a5a6eb5..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class Remote < Source - def content - strong_memoize(:content) do - next unless ci_config_path =~ URI::DEFAULT_PARSER.make_regexp(%w[http https]) - - YAML.dump('include' => [{ 'remote' => ci_config_path }]) - end - end - - def source - :remote_source - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/repository.rb b/lib/gitlab/ci/pipeline/chain/config/content/repository.rb deleted file mode 100644 index 0752b099d3d..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/repository.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class Repository < Source - def content - strong_memoize(:content) do - next unless file_in_repository? - - YAML.dump('include' => [{ 'local' => ci_config_path }]) - end - end - - def source - :repository_source - end - - private - - def file_in_repository? - return unless project - return unless @pipeline.sha - - project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path).present? - rescue GRPC::NotFound, GRPC::Internal - nil - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/source.rb b/lib/gitlab/ci/pipeline/chain/config/content/source.rb deleted file mode 100644 index 69dca1568b6..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/source.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - # When removing ci_project_pipeline_config_refactoring, this and its subclasses will be removed. - class Source - include Gitlab::Utils::StrongMemoize - - DEFAULT_YAML_FILE = '.gitlab-ci.yml' - - attr_reader :command - - def initialize(pipeline, command) - @pipeline = pipeline - @command = command - end - - def exists? - strong_memoize(:exists) do - content.present? - end - end - - def content - raise NotImplementedError - end - - def source - raise NotImplementedError - end - - def project - @project ||= @pipeline.project - end - - def ci_config_path - @ci_config_path ||= project.ci_config_path.presence || DEFAULT_YAML_FILE - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb b/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb new file mode 100644 index 00000000000..8b26416edf7 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Limit + class ActiveJobs < Chain::Base + include ::Gitlab::Utils::StrongMemoize + include ::Gitlab::Ci::Pipeline::Chain::Helpers + + LIMIT_NAME = :ci_active_jobs + MESSAGE = "Project exceeded the allowed number of jobs in active pipelines. Retry later." + + def perform! + return unless limits.exceeded?(LIMIT_NAME, count_jobs_in_alive_pipelines) + + error(MESSAGE, drop_reason: :job_activity_limit_exceeded) + + Gitlab::AppLogger.info( + class: self.class.name, + message: MESSAGE, + project_id: project.id, + plan: project.actual_plan_name) + end + + def break? + pipeline.errors.any? + end + + private + + def namespace + strong_memoize(:namespace) do + project.namespace + end + end + + def limits + strong_memoize(:limits) do + namespace.actual_limits + end + end + + def count_jobs_in_alive_pipelines + strong_memoize(:count_jobs_in_alive_pipelines) do + count_persisted_jobs_in_all_alive_pipelines + count_current_pipeline_jobs + end + end + + def count_current_pipeline_jobs + command.pipeline_seed.size + end + + def count_persisted_jobs_in_all_alive_pipelines + project.all_pipelines.jobs_count_in_alive_pipelines + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb b/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb deleted file mode 100644 index 3706dd0b9f6..00000000000 --- a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Limit - class JobActivity < Chain::Base - def perform! - # to be overridden in EE - end - - def break? - false # to be overridden in EE - end - end - end - end - end - end -end - -Gitlab::Ci::Pipeline::Chain::Limit::JobActivity.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::JobActivity') diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 654e24be8e1..4bec8355732 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -25,6 +25,8 @@ module Gitlab return error('Failed to build the pipeline!') end + set_pipeline_name + raise Populate::PopulateError if pipeline.persisted? end @@ -34,6 +36,15 @@ module Gitlab private + def set_pipeline_name + return if Feature.disabled?(:pipeline_name, pipeline.project) || + @command.yaml_processor_result.workflow_name.blank? + + name = @command.yaml_processor_result.workflow_name + + pipeline.build_pipeline_metadata(project: pipeline.project, title: name) + end + def stage_names # We filter out `.pre/.post` stages, as they alone are not considered # a complete pipeline: diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb index de24bbf688b..e8a991026b5 100644 --- a/lib/gitlab/ci/pipeline/duration.rb +++ b/lib/gitlab/ci/pipeline/duration.rb @@ -91,7 +91,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def from_pipeline(pipeline) status = %w[success failed running canceled] - builds = pipeline.builds.latest + builds = pipeline.processables.latest .where(status: status).where.not(started_at: nil).order(:started_at) from_builds(builds) diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index 44d905faced..4b7cbae5004 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -86,6 +86,7 @@ module Gitlab 'count' => values.size, 'min' => values.min, 'max' => values.max, + 'sum' => values.sum, 'avg' => values.sum / values.size } end.compact diff --git a/lib/gitlab/ci/processable_object_hierarchy.rb b/lib/gitlab/ci/processable_object_hierarchy.rb index 1122361e27e..c1531c3f4ab 100644 --- a/lib/gitlab/ci/processable_object_hierarchy.rb +++ b/lib/gitlab/ci/processable_object_hierarchy.rb @@ -20,12 +20,16 @@ module Gitlab def ancestor_conditions(cte) middle_table[:name].eq(objects_table[:name]).and( middle_table[:build_id].eq(cte.table[:id]) + ).and( + objects_table[:commit_id].eq(cte.table[:commit_id]) ) end def descendant_conditions(cte) middle_table[:build_id].eq(objects_table[:id]).and( middle_table[:name].eq(cte.table[:name]) + ).and( + objects_table[:commit_id].eq(cte.table[:commit_id]) ) end end diff --git a/lib/gitlab/ci/reports/sbom/source.rb b/lib/gitlab/ci/reports/sbom/source.rb index ea0fb8d4fbb..fbb8644c1b0 100644 --- a/lib/gitlab/ci/reports/sbom/source.rb +++ b/lib/gitlab/ci/reports/sbom/source.rb @@ -5,12 +5,11 @@ module Gitlab module Reports module Sbom class Source - attr_reader :source_type, :data, :fingerprint + attr_reader :source_type, :data - def initialize(type:, data:, fingerprint:) + def initialize(type:, data:) @source_type = type @data = data - @fingerprint = fingerprint end end end diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb index 70f2919d38d..54b21da5436 100644 --- a/lib/gitlab/ci/reports/security/report.rb +++ b/lib/gitlab/ci/reports/security/report.rb @@ -69,6 +69,10 @@ module Gitlab replace_with!(::Security::MergeReportsService.new(self, other).execute) end + def primary_identifiers + scanners.values.flat_map(&:primary_identifiers).compact + end + def primary_scanner scanners.first&.second end diff --git a/lib/gitlab/ci/reports/security/scanner.rb b/lib/gitlab/ci/reports/security/scanner.rb index 918df163ede..080ed3f834a 100644 --- a/lib/gitlab/ci/reports/security/scanner.rb +++ b/lib/gitlab/ci/reports/security/scanner.rb @@ -16,15 +16,16 @@ module Gitlab "semgrep" => 2 }.freeze - attr_accessor :external_id, :name, :vendor, :version + attr_accessor :external_id, :name, :vendor, :version, :primary_identifiers alias_method :key, :external_id - def initialize(external_id:, name:, vendor:, version:) + def initialize(external_id:, name:, vendor:, version:, primary_identifiers: nil) @external_id = external_id @name = name @vendor = vendor @version = version + @primary_identifiers = primary_identifiers end def to_hash diff --git a/lib/gitlab/ci/secure_files/cer.rb b/lib/gitlab/ci/secure_files/cer.rb new file mode 100644 index 00000000000..45d2898c29b --- /dev/null +++ b/lib/gitlab/ci/secure_files/cer.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module SecureFiles + class Cer + include Gitlab::Utils::StrongMemoize + + attr_reader :error + + def initialize(filedata) + @filedata = filedata + @error = nil + end + + def certificate_data + OpenSSL::X509::Certificate.new(@filedata) + rescue OpenSSL::X509::CertificateError => err + @error = err.to_s + nil + end + strong_memoize_attr :certificate_data + + def metadata + return {} unless certificate_data + + { + issuer: issuer, + subject: subject, + id: id, + expires_at: expires_at + } + end + strong_memoize_attr :metadata + + private + + def expires_at + certificate_data.not_before + end + + def id + certificate_data.serial.to_s + end + + def issuer + X509Name.parse(certificate_data.issuer) + end + + def subject + X509Name.parse(certificate_data.subject) + end + end + end + end +end diff --git a/lib/gitlab/ci/secure_files/mobile_provision.rb b/lib/gitlab/ci/secure_files/mobile_provision.rb new file mode 100644 index 00000000000..4ea74e20310 --- /dev/null +++ b/lib/gitlab/ci/secure_files/mobile_provision.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true +require 'cfpropertylist' + +module Gitlab + module Ci + module SecureFiles + class MobileProvision + include Gitlab::Utils::StrongMemoize + + attr_reader :error + + def initialize(filedata) + @filedata = filedata + end + + def decoded_plist + p7 = OpenSSL::PKCS7.new(@filedata) + p7.verify(nil, OpenSSL::X509::Store.new, nil, OpenSSL::PKCS7::NOVERIFY) + p7.data + rescue ArgumentError, OpenSSL::PKCS7::PKCS7Error => err + @error = err.to_s + nil + end + strong_memoize_attr :decoded_plist + + def properties + list = CFPropertyList::List.new(data: decoded_plist, format: CFPropertyList::List::FORMAT_XML).value + CFPropertyList.native_types(list) + rescue CFFormatError, CFPlistError, CFTypeError => err + @error = err.to_s + nil + end + strong_memoize_attr :properties + + def metadata + return {} unless properties + + { + id: id, + expires_at: expires_at, + platforms: properties["Platform"], + team_name: properties['TeamName'], + team_id: properties['TeamIdentifier'], + app_name: properties['AppIDName'], + app_id: properties['Name'], + app_id_prefix: properties['ApplicationIdentifierPrefix'], + xcode_managed: properties['IsXcodeManaged'], + entitlements: properties['Entitlements'], + devices: properties['ProvisionedDevices'], + certificate_ids: certificate_ids + } + end + strong_memoize_attr :metadata + + private + + def id + properties['UUID'] + end + + def expires_at + properties['ExpirationDate'] + end + + def certificate_ids + return [] if developer_certificates.empty? + + developer_certificates.map { |c| c.metadata[:id] } + end + + def developer_certificates + certificates = properties['DeveloperCertificates'] + return if certificates.empty? + + certs = [] + certificates.each_with_object([]) do |cert, obj| + certs << Cer.new(cert) + end + + certs + end + end + end + end +end diff --git a/lib/gitlab/ci/secure_files/p12.rb b/lib/gitlab/ci/secure_files/p12.rb new file mode 100644 index 00000000000..1006a4d05b2 --- /dev/null +++ b/lib/gitlab/ci/secure_files/p12.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module SecureFiles + class P12 + include Gitlab::Utils::StrongMemoize + + attr_reader :error + + def initialize(filedata, password = nil) + @filedata = filedata + @password = password + end + + def certificate_data + OpenSSL::PKCS12.new(@filedata, @password).certificate + rescue OpenSSL::PKCS12::PKCS12Error => err + @error = err.to_s + nil + end + strong_memoize_attr :certificate_data + + def metadata + return {} unless certificate_data + + { + issuer: issuer, + subject: subject, + id: serial, + expires_at: expires_at + } + end + strong_memoize_attr :metadata + + private + + def expires_at + certificate_data.not_before + end + + def serial + certificate_data.serial.to_s + end + + def issuer + X509Name.parse(certificate_data.issuer) + end + + def subject + X509Name.parse(certificate_data.subject) + end + end + end + end +end diff --git a/lib/gitlab/ci/secure_files/x509_name.rb b/lib/gitlab/ci/secure_files/x509_name.rb new file mode 100644 index 00000000000..659959b8ae5 --- /dev/null +++ b/lib/gitlab/ci/secure_files/x509_name.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module SecureFiles + class X509Name + def self.parse(x509_name) + x509_name.to_utf8.split(',').to_h { |a| a.split('=') } + rescue StandardError + {} + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml index 3096af1b173..fbdaeecca5d 100644 --- a/lib/gitlab/ci/templates/C++.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -33,3 +33,8 @@ test: stage: test script: - ./runmytests.sh + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml index 4fe37ceaeaa..3379ce2f649 100644 --- a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -28,3 +28,8 @@ test: # If you need to run any migrations or configure the database, this # would be the point to do it. - lein test + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml index 68b55b782cd..9584ec5deef 100644 --- a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -42,3 +42,8 @@ spec: minitest: script: - crystal test/spec_test.cr # change to the file(s) you execute for tests + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml index acc4a9d2917..21dda92257e 100644 --- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -74,3 +74,8 @@ django-tests: - echo "GRANT ALL on *.* to '${MYSQL_USER}';"| mysql -u root --password="${MYSQL_ROOT_PASSWORD}" -h mysql # use python3 explicitly. see https://wiki.ubuntu.com/Python/3 - python3 manage.py test + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml index 83ddce936e6..8b20c4cbccc 100644 --- a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -24,3 +24,8 @@ before_script: mix: script: - mix test + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml index 021662ab416..7f81755348c 100644 --- a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -35,3 +35,8 @@ test: - $CI_PROJECT_DIR/coverage reports: junit: report.xml + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml index 603aede4d46..8cfea3e236f 100644 --- a/lib/gitlab/ci/templates/Go.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -28,3 +28,8 @@ compile: artifacts: paths: - mybinaries + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml index 08dc10d34b7..671925c5df6 100644 --- a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -1,3 +1,5 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: @@ -39,3 +41,8 @@ test: paths: - build - .gradle + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index 03c8941169f..01697f67b89 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -46,3 +46,8 @@ before_script: build: script: - ./gradlew build + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index ce227bad19a..071eccbab0d 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.14.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.19.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index ce227bad19a..071eccbab0d 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.14.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.19.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index e278539d214..23efed212f8 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:0.85.29" + CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:0.87.0" needs: [] script: - export SOURCE_CODE=$PWD @@ -26,6 +26,11 @@ code_quality: echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " done } + - | + if [ -n "$CODECLIMATE_REGISTRY_USERNAME" ] && [ -n "$CODECLIMATE_REGISTRY_PASSWORD" ] && [ -n "$CODECLIMATE_PREFIX" ]; then + CODECLIMATE_REGISTRY=${CODECLIMATE_PREFIX%%/*} + docker login "$CODECLIMATE_REGISTRY" --username "$CODECLIMATE_REGISTRY_USERNAME" --password "$CODECLIMATE_REGISTRY_PASSWORD" + fi - docker pull --quiet "$CODE_QUALITY_IMAGE" - | docker run --rm \ @@ -38,6 +43,8 @@ code_quality: REPORT_FORMAT \ ENGINE_MEMORY_LIMIT_BYTES \ CODECLIMATE_PREFIX \ + CODECLIMATE_REGISTRY_USERNAME \ + CODECLIMATE_REGISTRY_PASSWORD \ ) \ --volume "$PWD":/code \ --volume /var/run/docker.sock:/var/run/docker.sock \ diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 539e1a6385d..d994ed70ea9 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.37.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.39.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index 7cbc8e40b47..222f534387a 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -50,6 +50,8 @@ dependency_scanning: artifacts: paths: - "**/gl-sbom-*.cdx.json" + reports: + cyclonedx: "**/gl-sbom-*.cdx.json" .gemnasium-shared-rule: exists: diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml index 70f85382967..67057e916a8 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml @@ -50,6 +50,8 @@ dependency_scanning: artifacts: paths: - "**/gl-sbom-*.cdx.json" + reports: + cyclonedx: "**/gl-sbom-*.cdx.json" .gemnasium-shared-rule: exists: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 78fe108e8b9..7ad71625436 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.37.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.39.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index bc2e1fed0d4..10c843f60a6 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.37.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.39.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml index 34084272b29..d39b329c4b6 100644 --- a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -81,3 +81,8 @@ pages: # description page. # # [3]: https://hub.docker.com/_/julia/ + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index 3a490012f3d..7bb2ea328e7 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -78,3 +78,8 @@ test: # set it in your package.json script # comment this out if you don't have a frontend test - npm test + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index dfa46d7af61..347b811bc5d 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -14,26 +14,44 @@ # * Deploy built artifacts from master branch only. variables: - # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log. # `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work. - MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" - # As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used + MAVEN_OPTS: >- + -Dhttps.protocols=TLSv1.2 + -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository + -Dorg.slf4j.simpleLogger.showDateTime=true + -Djava.awt.headless=true + + # As of Maven 3.3.0 instead of this you MAY define these options in `.mvn/maven.config` so the same config is used # when running from the command line. + # As of Maven 3.6.1, the use of `--no-tranfer-progress` (or `-ntp`) suppresses download and upload messages. The use + # of the `Slf4jMavenTransferListener` is no longer necessary. # `installAtEnd` and `deployAtEnd` are only effective with recent version of the corresponding plugins. - MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true" + MAVEN_CLI_OPTS: >- + --batch-mode + --errors + --fail-at-end + --show-version + --no-transfer-progress + -DinstallAtEnd=true + -DdeployAtEnd=true -# This template uses jdk8 for verifying and deploying images -image: maven:3.3.9-jdk-8 +# This template uses the latest Maven 3 release, e.g., 3.8.6, and OpenJDK 8 (LTS) +# for verifying and deploying images +# Maven 3.8.x REQUIRES HTTPS repositories. +# See https://maven.apache.org/docs/3.8.1/release-notes.html#how-to-fix-when-i-get-a-http-repository-blocked for more. +image: maven:3-openjdk-8 # Cache downloaded dependencies and plugins between builds. # To keep cache across branches add 'key: "$CI_JOB_NAME"' +# Be aware that `mvn deploy` will install the built jar into this repository. If you notice your cache size +# increasing, consider adding `-Dmaven.install.skip=true` to `MAVEN_OPTS` or in `.mvn/maven.config` cache: paths: - .m2/repository # For merge requests do not `deploy` but only run `verify`. # See https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html -.verify: &verify +.verify: stage: test script: - 'mvn $MAVEN_CLI_OPTS verify' @@ -43,19 +61,20 @@ cache: # Verify merge requests using JDK8 verify:jdk8: - <<: *verify + extends: + - .verify -# To deploy packages from CI, create a ci_settings.xml file +# To deploy packages from CI, create a `ci_settings.xml` file # For deploying packages to GitLab's Maven Repository: See https://docs.gitlab.com/ee/user/packages/maven_repository/index.html#create-maven-packages-with-gitlab-cicd for more details. # Please note: The GitLab Maven Repository is currently only available in GitLab Premium / Ultimate. -# For `master` branch run `mvn deploy` automatically. +# For `master` or `main` branch run `mvn deploy` automatically. deploy:jdk8: stage: deploy script: - - if [ ! -f ci_settings.xml ]; - then echo "CI settings missing\! If deploying to GitLab Maven Repository, please see https://docs.gitlab.com/ee/user/packages/maven_repository/index.html#create-maven-packages-with-gitlab-cicd for instructions."; + - if [ ! -f ci_settings.xml ]; then + echo "CI settings missing\! If deploying to GitLab Maven Repository, please see https://docs.gitlab.com/ee/user/packages/maven_repository/index.html#create-maven-packages-with-gitlab-cicd for instructions."; fi - - 'mvn $MAVEN_CLI_OPTS deploy -s ci_settings.xml' + - 'mvn $MAVEN_CLI_OPTS deploy --settings ci_settings.xml' only: variables: - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml index 65db649e22f..e34bb8307f4 100644 --- a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -49,3 +49,8 @@ debug: - msbuild /p:Configuration="Debug" /p:Platform="Any CPU" /p:OutputPath="./../../build/debug/" "MyProject.sln" - mono packages/NUnit.ConsoleRunner.3.6.0/tools/nunit3-console.exe build/debug/MyProject.Test.dll + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml index 7a4f7ed628b..649d525df76 100644 --- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -33,3 +33,8 @@ test_db: script: - npm install - node ./specs/start.js ./specs/db-postgres.spec.js + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml index 12640d28d29..0604438e0aa 100644 --- a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -42,3 +42,8 @@ variables: test: script: - vendor/bin/phpunit --configuration phpunit.xml --coverage-text --colors=never + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml index 55cf22b6601..51d2273d41d 100644 --- a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -18,3 +18,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml index 2f518d667a5..e577a489c55 100644 --- a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml @@ -11,6 +11,7 @@ pages: - apk update && apk add doxygen - doxygen doxygen/Doxyfile - mv doxygen/documentation/html/ public/ + environment: production artifacts: paths: - public diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml index 9da50439be8..88ed73b41e7 100644 --- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -20,3 +20,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index 85f90984045..9f3ba6d5dd4 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -6,6 +6,7 @@ # Full project: https://gitlab.com/pages/plain-html pages: stage: deploy + environment: production script: - mkdir .public - cp -r ./* .public diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml index 9e48ac9fcdc..aa86ad2a6ad 100644 --- a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -18,3 +18,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml index a6f94a4d80e..b1617e9239c 100644 --- a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -11,6 +11,7 @@ pages: - npm install hexo-cli -g - test -e package.json && npm install - hexo generate + environment: production artifacts: paths: - public diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index 591eebf9cd6..d6f6e94526e 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -27,3 +27,4 @@ pages: only: variables: - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml index 59e55efaee0..fba4afca9ed 100644 --- a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml @@ -21,6 +21,7 @@ test: pages: stage: deploy + environment: production script: - pip install hyde - hyde gen -d public diff --git a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml index 8e15570fd1a..57e3ced4dc2 100644 --- a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml @@ -29,6 +29,7 @@ before_script: # This build job produced the output directory of your site pages: + environment: production script: - jbake . public artifacts: diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index e0ad2e55f7d..8b07454af24 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -36,3 +36,4 @@ pages: only: variables: - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml index 26fac92d0dc..ad083fcc5db 100644 --- a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -40,3 +40,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml index 9b5c1198c6c..e86337ae23c 100644 --- a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml @@ -15,3 +15,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml index d97f0b7beb7..a49e95b62c8 100644 --- a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -19,3 +19,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml index 17ce0ef3659..d8f036ab4ed 100644 --- a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -31,3 +31,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml index a3ce96da244..b0511abd109 100644 --- a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -15,3 +15,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml index 4abdf66a21c..c89050eede7 100644 --- a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -18,3 +18,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml index 7d52a407848..3721344b21e 100644 --- a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml @@ -13,3 +13,4 @@ pages: artifacts: paths: - public/ + environment: production diff --git a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml index 961941ac4d0..00efcfa1b32 100644 --- a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml @@ -32,3 +32,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index 191d5b6b11c..febbb36d834 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -55,3 +55,8 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml index a83f84da818..b9823444db2 100644 --- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -40,3 +40,8 @@ test:cargo: # when: always # reports: # junit: $CI_PROJECT_DIR/tests/*.xml + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml index 26efe7a8908..ce5d5937896 100644 --- a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -30,3 +30,8 @@ test: script: # Execute your project's tests - sbt clean test + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 8d6c191edc4..f12efa1db34 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -40,6 +40,19 @@ apifuzzer_fuzz: - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DAST_API_IMAGE_SUFFIX: "-fips" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. - if: $CI_COMMIT_BRANCH && $CI_GITLAB_FIPS_MODE == "true" variables: @@ -55,5 +68,3 @@ apifuzzer_fuzz: - gl-*.log reports: api_fuzzing: gl-api-fuzzing-report.json - -# end diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index d4b6a252b25..d933007ec61 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -31,15 +31,15 @@ coverage_fuzzing_unlicensed: stage: fuzz allow_failure: true before_script: - - export COVFUZZ_JOB_TOKEN=$CI_JOB_TOKEN - - export COVFUZZ_PRIVATE_TOKEN=$CI_PRIVATE_TOKEN - - export COVFUZZ_PROJECT_PATH=$CI_PROJECT_PATH - - export COVFUZZ_PROJECT_ID=$CI_PROJECT_ID + - export COVFUZZ_JOB_TOKEN="$CI_JOB_TOKEN" + - export COVFUZZ_PRIVATE_TOKEN="$CI_PRIVATE_TOKEN" + - export COVFUZZ_PROJECT_PATH="$CI_PROJECT_PATH" + - export COVFUZZ_PROJECT_ID="$CI_PROJECT_ID" - if [ -x "$(command -v apt-get)" ] ; then apt-get update && apt-get install -y wget; fi - wget -O gitlab-cov-fuzz "${COVFUZZ_URL_PREFIX}"/"${COVFUZZ_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64 - chmod a+x gitlab-cov-fuzz - export REGRESSION=true - - if [[ $CI_COMMIT_BRANCH = $COVFUZZ_BRANCH ]]; then REGRESSION=false; fi; + - if [[ "$CI_COMMIT_BRANCH" = "$COVFUZZ_BRANCH" ]]; then REGRESSION=false; fi; artifacts: paths: - corpus diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml new file mode 100644 index 00000000000..feed4c47157 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml @@ -0,0 +1,64 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml + +# Read more about this feature https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing +# +# Configure coverage fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/#available-cicd-variables + +variables: + # Which branch we want to run full fledged long running fuzzing jobs. + # All others will run fuzzing regression + COVFUZZ_BRANCH: "$CI_DEFAULT_BRANCH" + # This is using semantic version and will always download latest v3 gitlab-cov-fuzz release + COVFUZZ_VERSION: v3 + # This is for users who have an offline environment and will have to replicate gitlab-cov-fuzz release binaries + # to their own servers + COVFUZZ_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw" + + +coverage_fuzzing_unlicensed: + stage: .pre + allow_failure: true + rules: + - if: $GITLAB_FEATURES !~ /\bcoverage_fuzzing\b/ && $COVFUZZ_DISABLED == null + script: + - echo "ERROR Your GitLab project is missing licensing for Coverage Fuzzing" && exit 1 + +.fuzz_base: + stage: fuzz + allow_failure: true + before_script: + - export COVFUZZ_JOB_TOKEN="$CI_JOB_TOKEN" + - export COVFUZZ_PRIVATE_TOKEN="$CI_PRIVATE_TOKEN" + - export COVFUZZ_PROJECT_PATH="$CI_PROJECT_PATH" + - export COVFUZZ_PROJECT_ID="$CI_PROJECT_ID" + - if [ -x "$(command -v apt-get)" ] ; then apt-get update && apt-get install -y wget; fi + - wget -O gitlab-cov-fuzz "${COVFUZZ_URL_PREFIX}"/"${COVFUZZ_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64 + - chmod a+x gitlab-cov-fuzz + - export REGRESSION=true + - if [[ "$CI_COMMIT_BRANCH" = "$COVFUZZ_BRANCH" ]]; then REGRESSION=false; fi; + artifacts: + paths: + - corpus + - crashes + - gl-coverage-fuzzing-report.json + reports: + coverage_fuzzing: gl-coverage-fuzzing-report.json + when: always + rules: + - if: $COVFUZZ_DISABLED + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/ + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/ diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml index 8aabf20c5df..a28914d082f 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml @@ -40,6 +40,19 @@ dast_api: - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DAST_API_IMAGE_SUFFIX: "-fips" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. - if: $CI_COMMIT_BRANCH && $CI_GITLAB_FIPS_MODE == "true" variables: diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index 9d3b1f4316e..50e9bb5431d 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -52,6 +52,19 @@ dast: - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && $REVIEW_DISABLED when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && + $GITLAB_FEATURES =~ /\bdast\b/ + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdast\b/ + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. - if: $CI_COMMIT_BRANCH && ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && $GITLAB_FEATURES =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml index 3c4533d603e..2a5ac539a42 100644 --- a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -38,3 +38,8 @@ archive_project: - ios_11-3 - xcode_9-3 - macos_10-13 + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml index b8d284532bd..5fcbb251672 100644 --- a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml @@ -111,3 +111,8 @@ tests: # (e.g. integration tests, unit tests etc). script: - 'dotnet test --no-restore' + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml index 58e840da713..400226b415d 100644 --- a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml @@ -36,3 +36,8 @@ build: artifacts: paths: - ./*.ipa + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index c5664ef1cfb..2dc7bbc391e 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -140,7 +140,7 @@ module Gitlab def being_watched? Gitlab::Redis::SharedState.with do |redis| - redis.exists(being_watched_cache_key) + redis.exists?(being_watched_cache_key) # rubocop:disable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 528d72c9bcc..cf5f04215ad 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -11,6 +11,7 @@ module Gitlab @instance_variables_builder = Builder::Instance.new @project_variables_builder = Builder::Project.new(project) @group_variables_builder = Builder::Group.new(project&.group) + @release_variables_builder = Builder::Release.new(release) end def scoped_variables(job, environment:, dependencies:) @@ -28,6 +29,7 @@ module Gitlab variables.concat(secret_project_variables(environment: environment)) variables.concat(pipeline.variables) variables.concat(pipeline_schedule_variables) + variables.concat(release_variables) end end @@ -106,18 +108,26 @@ module Gitlab end end + def release_variables + strong_memoize(:release_variables) do + release_variables_builder.variables + end + end + private attr_reader :pipeline attr_reader :instance_variables_builder attr_reader :project_variables_builder attr_reader :group_variables_builder + attr_reader :release_variables_builder delegate :project, to: :pipeline def predefined_variables(job) Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI_JOB_NAME', value: job.name) + variables.append(key: 'CI_JOB_NAME_SLUG', value: job_name_slug(job)) variables.append(key: 'CI_JOB_STAGE', value: job.stage_name) variables.append(key: 'CI_JOB_MANUAL', value: 'true') if job.action? variables.append(key: 'CI_PIPELINE_TRIGGERED', value: 'true') if job.trigger_request @@ -145,6 +155,10 @@ module Gitlab end end + def job_name_slug(job) + job.name && Gitlab::Utils.slugify(job.name) + end + def ci_node_total_value(job) parallel = job.options&.dig(:parallel) parallel = parallel.dig(:total) if parallel.is_a?(Hash) @@ -166,6 +180,12 @@ module Gitlab container[args] = yield end end + + def release + return unless @pipeline.tag? + + project.releases.find_by_tag(@pipeline.ref) + end end end end diff --git a/lib/gitlab/ci/variables/builder/release.rb b/lib/gitlab/ci/variables/builder/release.rb new file mode 100644 index 00000000000..25f3d8446c5 --- /dev/null +++ b/lib/gitlab/ci/variables/builder/release.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Builder + class Release + include Gitlab::Utils::StrongMemoize + + attr_reader :release + + DESCRIPTION_LIMIT = 1024 + + def initialize(release) + @release = release + end + + def variables + strong_memoize(:variables) do + ::Gitlab::Ci::Variables::Collection.new.tap do |variables| + next variables unless release + + if release.description + variables.append( + key: 'CI_RELEASE_DESCRIPTION', + value: release.description.truncate(DESCRIPTION_LIMIT), + raw: true) + end + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index 52673d03e69..b6d6e1a3e5f 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -72,7 +72,7 @@ module Gitlab Collection.new(@variables.reject(&block)) end - def expand_value(value, keep_undefined: false, expand_file_vars: true) + def expand_value(value, keep_undefined: false, expand_file_vars: true, project: nil) value.gsub(Item::VARIABLES_REGEXP) do match = Regexp.last_match # it is either a valid variable definition or a ($$ / %%) full_match = match[0] @@ -88,6 +88,16 @@ module Gitlab if variable # VARIABLE_NAME is an existing variable next variable.value unless variable.file? + # Will be cleaned up with https://gitlab.com/gitlab-org/gitlab/-/issues/378266 + if project + # We only log if `project` exists to make sure it is called from `Ci::BuildRunnerPresenter` + # when the variables are sent to Runner. + Gitlab::AppJsonLogger.info( + event: 'file_variable_is_referenced_in_another_variable', + project_id: project.id + ) + end + expand_file_vars ? variable.value : full_match elsif keep_undefined full_match # we do not touch the variable definition @@ -97,7 +107,7 @@ module Gitlab end end - def sort_and_expand_all(keep_undefined: false, expand_file_vars: true) + def sort_and_expand_all(keep_undefined: false, expand_file_vars: true, project: nil) sorted = Sort.new(self) return self.class.new(self, sorted.errors) unless sorted.valid? @@ -112,7 +122,8 @@ module Gitlab # expand variables as they are added variable = item.to_runner_variable variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined, - expand_file_vars: expand_file_vars) + expand_file_vars: expand_file_vars, + project: project) new_collection.append(variable) end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index f203f88442d..5c3864362da 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -32,18 +32,16 @@ module Gitlab end end - def stage_builds_attributes(stage) - jobs.values - .select { |job| job[:stage] == stage } - .map { |job| build_attributes(job[:name]) } - end - def workflow_rules @workflow_rules ||= @ci_config.workflow_rules end + def workflow_name + @workflow_name ||= @ci_config.workflow_name&.strip + end + def root_variables - @root_variables ||= transform_to_array(variables) + @root_variables ||= transform_to_array(@ci_config.variables) end def jobs @@ -58,6 +56,38 @@ module Gitlab @included_templates ||= @ci_config.included_templates end + def variables_with_data + @ci_config.variables_with_data + end + + def yaml_variables_for(job_name) + job = jobs[job_name] + + return [] unless job + + Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( + from: root_variables, + to: job[:job_variables], + inheritance: job.fetch(:root_variables_inheritance, true) + ) + end + + def stage_for(job_name) + jobs.dig(job_name, :stage) + end + + def config_metadata + @ci_config&.metadata || {} + end + + private + + def stage_builds_attributes(stage) + jobs.values + .select { |job| job[:stage] == stage } + .map { |job| build_attributes(job[:name]) } + end + def build_attributes(name) job = jobs.fetch(name.to_sym, {}) @@ -103,36 +133,6 @@ module Gitlab }.compact }.compact end - def variables_with_data - @ci_config.variables_with_data - end - - def yaml_variables_for(job_name) - job = jobs[job_name] - - return [] unless job - - Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( - from: root_variables, - to: job[:job_variables], - inheritance: job.fetch(:root_variables_inheritance, true) - ) - end - - def stage_for(job_name) - jobs.dig(job_name, :stage) - end - - def config_metadata - @ci_config&.metadata || {} - end - - private - - def variables - @variables ||= @ci_config.variables - end - def release(job) job[:release] end diff --git a/lib/gitlab/config/entry/legacy_validation_helpers.rb b/lib/gitlab/config/entry/legacy_validation_helpers.rb index be7d26fed4e..415f6f77214 100644 --- a/lib/gitlab/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/config/entry/legacy_validation_helpers.rb @@ -50,12 +50,6 @@ module Gitlab variables.values.flatten(1).all?(&method(:validate_alphanumeric)) end - def validate_string_or_hash_value_variables(variables, allowed_value_data) - variables.is_a?(Hash) && - variables.keys.all?(&method(:validate_alphanumeric)) && - variables.values.all? { |value| validate_string_or_hash_value_variable(value, allowed_value_data) } - end - def validate_alphanumeric(value) validate_string(value) || validate_integer(value) end @@ -68,14 +62,6 @@ module Gitlab value.is_a?(String) || value.is_a?(Symbol) end - def validate_string_or_hash_value_variable(value, allowed_value_data) - if value.is_a?(Hash) - (value.keys - allowed_value_data).empty? && value.values.all?(&method(:validate_alphanumeric)) - else - validate_alphanumeric(value) - end - end - def validate_regexp(value) Gitlab::UntrustedRegexp::RubySyntax.valid?(value) end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 337cfbc5287..b88a6766d92 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -44,7 +44,7 @@ module Gitlab mutually_exclusive_keys = value.try(:keys).to_a & options[:in] if mutually_exclusive_keys.length > 1 - record.errors.add(attribute, "please use only one the following keys: " + + record.errors.add(attribute, "please use only one of the following keys: " + mutually_exclusive_keys.join(', ')) end end @@ -304,15 +304,12 @@ module Gitlab end end - # This will be removed with the FF `ci_variables_refactoring_to_variable`. class VariablesValidator < ActiveModel::EachValidator include LegacyValidationHelpers def validate_each(record, attribute, value) if options[:array_values] validate_key_array_values(record, attribute, value) - elsif options[:allowed_value_data] - validate_key_hash_values(record, attribute, value, options[:allowed_value_data]) else validate_key_values(record, attribute, value) end @@ -329,12 +326,6 @@ module Gitlab record.errors.add(attribute, 'should be a hash of key value pairs, value can be an array') end end - - def validate_key_hash_values(record, attribute, value, allowed_value_data) - unless validate_string_or_hash_value_variables(value, allowed_value_data) - record.errors.add(attribute, 'should be a hash of key value pairs, value can be a hash') - end - end end class AlphanumericValidator < ActiveModel::EachValidator diff --git a/lib/gitlab/config_checker/external_database_checker.rb b/lib/gitlab/config_checker/external_database_checker.rb index 54320b7ff9a..64950fb4eef 100644 --- a/lib/gitlab/config_checker/external_database_checker.rb +++ b/lib/gitlab/config_checker/external_database_checker.rb @@ -5,22 +5,29 @@ module Gitlab module ExternalDatabaseChecker extend self + PG_REQUIREMENTS_LINK = + '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">database requirements</a>' + def check - return [] if ApplicationRecord.database.postgresql_minimum_supported_version? + unsupported_database = Gitlab::Database + .database_base_models + .map { |_, model| Gitlab::Database::Reflection.new(model) } + .reject(&:postgresql_minimum_supported_version?) - [ + unsupported_database.map do |database| { type: 'warning', message: _('You are using PostgreSQL %{pg_version_current}, but PostgreSQL ' \ '%{pg_version_minimum} is required for this version of GitLab. ' \ 'Please upgrade your environment to a supported PostgreSQL version, ' \ - 'see %{pg_requirements_url} for details.') % { - pg_version_current: ApplicationRecord.database.version, - pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION, - pg_requirements_url: '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">database requirements</a>' - } + 'see %{pg_requirements_url} for details.') % \ + { + pg_version_current: database.version, + pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION, + pg_requirements_url: PG_REQUIREMENTS_LINK + } } - ] + end end end end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 320ebe5e80f..a75c7c539ae 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -63,6 +63,7 @@ module Gitlab def hook_attrs(pipeline) { id: pipeline.id, + iid: pipeline.iid, ref: pipeline.source_ref, tag: pipeline.tag, sha: pipeline.sha, diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 45f52765d0f..92cafd1d00e 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -67,11 +67,11 @@ module Gitlab end event :finish do - transition any => :finished + transition [:paused, :finished, :active, :finalizing] => :finished end event :failure do - transition any => :failed + transition [:failed, :finalizing, :active] => :failed end event :finalize do diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 5725d7a4503..c4a9cf8b80f 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -99,6 +99,7 @@ ci_pipeline_messages: :gitlab_ci ci_pipeline_schedules: :gitlab_ci ci_pipeline_schedule_variables: :gitlab_ci ci_pipelines_config: :gitlab_ci +ci_pipeline_metadata: :gitlab_ci ci_pipelines: :gitlab_ci ci_pipeline_variables: :gitlab_ci ci_platform_metrics: :gitlab_ci @@ -263,6 +264,8 @@ incident_management_oncall_shifts: :gitlab_main incident_management_pending_alert_escalations: :gitlab_main incident_management_pending_issue_escalations: :gitlab_main incident_management_timeline_events: :gitlab_main +incident_management_timeline_event_tags: :gitlab_main +incident_management_timeline_event_tag_links: :gitlab_main index_statuses: :gitlab_main in_product_marketing_emails: :gitlab_main insights: :gitlab_main @@ -389,6 +392,7 @@ packages_nuget_dependency_link_metadata: :gitlab_main packages_nuget_metadata: :gitlab_main packages_package_file_build_infos: :gitlab_main packages_package_files: :gitlab_main +packages_rpm_repository_files: :gitlab_main packages_packages: :gitlab_main packages_pypi_metadata: :gitlab_main packages_rubygems_metadata: :gitlab_main @@ -447,6 +451,7 @@ projects: :gitlab_main projects_sync_events: :gitlab_main project_statistics: :gitlab_main project_topics: :gitlab_main +project_wiki_repository_states: :gitlab_main prometheus_alert_events: :gitlab_main prometheus_alerts: :gitlab_main prometheus_metrics: :gitlab_main @@ -545,6 +550,7 @@ user_group_callouts: :gitlab_main user_project_callouts: :gitlab_main user_highest_roles: :gitlab_main user_interacted_projects: :gitlab_main +user_phone_number_validations: :gitlab_main user_permission_export_uploads: :gitlab_main user_preferences: :gitlab_main users: :gitlab_main diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 40b76a1c028..0881025b425 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -105,16 +105,25 @@ module Gitlab def read_write connection = nil transaction_open = nil + attempts = 3 + + if prevent_load_balancer_retries_in_transaction? + attempts = 1 if pool.connection.transaction_open? + end + # In the event of a failover the primary may be briefly unavailable. # Instead of immediately grinding to a halt we'll retry the operation # a few times. - retry_with_backoff do + # It is not possible preserve transaction state during a retry, so we do not retry in that case. + retry_with_backoff(attempts: attempts) do |attempt| connection = pool.connection transaction_open = connection.transaction_open? yield connection rescue StandardError => e - if transaction_open && connection_error?(e) + # No leaking will happen on the final attempt. Leaks are caused by subsequent retries + not_final_attempt = attempt && attempt < attempts + if transaction_open && connection_error?(e) && not_final_attempt ::Gitlab::Database::LoadBalancing::Logger.warn( event: :transaction_leak, message: 'A write transaction has leaked during database fail-over' @@ -171,7 +180,7 @@ module Gitlab end # Yields a block, retrying it upon error using an exponential backoff. - def retry_with_backoff(retries = 3, time = 2) + def retry_with_backoff(attempts: 3, time: 2) # In CI we only use the primary, but databases may not always be # available (or take a few seconds to become available). Retrying in # this case can slow down CI jobs. In addition, retrying with _only_ @@ -183,12 +192,12 @@ module Gitlab # replicas were configured. return yield if primary_only? - retried = 0 + attempt = 1 last_error = nil - while retried < retries + while attempt <= attempts begin - return yield + return yield attempt # Yield the current attempt count rescue StandardError => error raise error unless connection_error?(error) @@ -198,7 +207,7 @@ module Gitlab last_error = error sleep(time) - retried += 1 + attempt += 1 time **= 2 end end @@ -332,6 +341,10 @@ module Gitlab row = ar_connection.select_all(sql).first row['location'] if row end + + def prevent_load_balancer_retries_in_transaction? + Gitlab::Utils.to_boolean(ENV['PREVENT_LOAD_BALANCER_RETRIES_IN_TRANSACTION'], default: false) + end end end end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index 5d91292b8de..3180289ec69 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -97,8 +97,14 @@ module Gitlab end def databases_in_sync?(wal_locations) + locations = if Feature.enabled?(:indifferent_wal_location_keys) + wal_locations.with_indifferent_access + else + wal_locations + end + ::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb| - if (location = wal_locations[lb.name]) + if (location = locations[lb.name]) lb.select_up_to_date_host(location) else # If there's no entry for a load balancer it means the Sidekiq diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index e574422ce11..df40e3b3868 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -296,12 +296,11 @@ module Gitlab with_lock_retries do execute("LOCK TABLE #{target}, #{source} IN SHARE ROW EXCLUSIVE MODE") if reverse_lock_order - execute <<-EOF.strip_heredoc ALTER TABLE #{source} ADD CONSTRAINT #{options[:name]} - FOREIGN KEY (#{options[:column]}) - REFERENCES #{target} (#{target_column}) + FOREIGN KEY (#{multiple_columns(options[:column])}) + REFERENCES #{target} (#{multiple_columns(target_column)}) #{on_delete_statement(options[:on_delete])} NOT VALID; EOF @@ -355,7 +354,7 @@ module Gitlab # - For standard rails foreign keys the prefix is `fk_rails_` # def concurrent_foreign_key_name(table, column, prefix: 'fk_') - identifier = "#{table}_#{column}_fk" + identifier = "#{table}_#{multiple_columns(column, separator: '_')}_fk" hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) "#{prefix}#{hashed_identifier}" @@ -1503,6 +1502,26 @@ into similar problems in the future (e.g. when new tables are created). SQL end + def drop_constraint(table_name, constraint_name, cascade: false) + execute <<~SQL + ALTER TABLE #{quote_table_name(table_name)} DROP CONSTRAINT #{quote_column_name(constraint_name)} #{cascade_statement(cascade)} + SQL + end + + def add_primary_key_using_index(table_name, pk_name, index_to_use) + execute <<~SQL + ALTER TABLE #{quote_table_name(table_name)} ADD CONSTRAINT #{quote_table_name(pk_name)} PRIMARY KEY USING INDEX #{quote_table_name(index_to_use)} + SQL + end + + def swap_primary_key(table_name, primary_key_name, index_to_use) + with_lock_retries(raise_on_exhaustion: true) do + drop_constraint(table_name, primary_key_name, cascade: true) + add_primary_key_using_index(table_name, primary_key_name, index_to_use) + end + end + alias_method :unswap_primary_key, :swap_primary_key + def drop_sequence(table_name, column_name, sequence_name) execute <<~SQL ALTER TABLE #{quote_table_name(table_name)} ALTER COLUMN #{quote_column_name(column_name)} DROP DEFAULT; @@ -1519,6 +1538,14 @@ into similar problems in the future (e.g. when new tables are created). private + def multiple_columns(columns, separator: ', ') + Array.wrap(columns).join(separator) + end + + def cascade_statement(cascade) + cascade ? 'CASCADE' : '' + end + def create_temporary_columns_and_triggers(table, columns, primary_key: :id, data_type: :bigint) unless table_exists?(table) raise "Table #{table} does not exist" diff --git a/lib/gitlab/database/migrations/base_background_runner.rb b/lib/gitlab/database/migrations/base_background_runner.rb index 76982a9da9b..dbb85bad95c 100644 --- a/lib/gitlab/database/migrations/base_background_runner.rb +++ b/lib/gitlab/database/migrations/base_background_runner.rb @@ -4,10 +4,11 @@ module Gitlab module Database module Migrations class BaseBackgroundRunner - attr_reader :result_dir + attr_reader :result_dir, :connection - def initialize(result_dir:) + def initialize(result_dir:, connection:) @result_dir = result_dir + @connection = connection end def jobs_by_migration_name @@ -45,7 +46,7 @@ module Gitlab instrumentation.observe(version: nil, name: batch_names.next, - connection: ActiveRecord::Migration.connection) do + connection: connection) do run_job(j) end end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index 4404b5bf961..85dc6051c7c 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -6,40 +6,68 @@ module Gitlab class Runner BASE_RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze METADATA_FILENAME = 'metadata.json' - SCHEMA_VERSION = 3 # Version of the output format produced by the runner + SCHEMA_VERSION = 4 # Version of the output format produced by the runner class << self - def up - Runner.new(direction: :up, migrations: migrations_for_up, result_dir: BASE_RESULT_DIR.join('up')) + def up(database:, legacy_mode: false) + within_context_for_database(database) do + Runner.new(direction: :up, database: database, migrations: migrations_for_up(database), legacy_mode: legacy_mode) + end end - def down - Runner.new(direction: :down, migrations: migrations_for_down, result_dir: BASE_RESULT_DIR.join('down')) + def down(database:, legacy_mode: false) + within_context_for_database(database) do + Runner.new(direction: :down, database: database, migrations: migrations_for_down(database), legacy_mode: legacy_mode) + end end def background_migrations TestBackgroundRunner.new(result_dir: BASE_RESULT_DIR.join('background_migrations')) end - def batched_background_migrations(for_database:) + def batched_background_migrations(for_database:, legacy_mode: false) runner = nil + result_dir = if legacy_mode + BASE_RESULT_DIR.join('background_migrations') + else + BASE_RESULT_DIR.join(for_database.to_s, 'background_migrations') + end + # Only one loop iteration since we pass `only:` here Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| runner = Gitlab::Database::Migrations::TestBatchedBackgroundRunner - .new(result_dir: BASE_RESULT_DIR.join('background_migrations'), connection: connection) + .new(result_dir: result_dir, connection: connection) end runner end def migration_context - @migration_context ||= ApplicationRecord.connection.migration_context + # We're mirroring rails internal migration code, which requires that + # ActiveRecord::Base has connected to the current database. The correct database is chosen by + # within_context_for_database + ActiveRecord::Base.connection.migration_context # rubocop:disable Database/MultipleDatabases + end + + # rubocop:disable Database/MultipleDatabases + def within_context_for_database(database) + original_db_config = ActiveRecord::Base.connection_db_config + # The config only works if passed a string + db_config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: database.to_s) + raise ArgumentError, "Cannot find a database configuration for #{database}" unless db_config + + ActiveRecord::Base.establish_connection(db_config) # rubocop:disable Database/EstablishConnection + + yield + ensure + ActiveRecord::Base.establish_connection(original_db_config) # rubocop:disable Database/EstablishConnection end + # rubocop:enable Database/MultipleDatabases private - def migrations_for_up + def migrations_for_up(database) existing_versions = migration_context.get_all_versions.to_set migration_context.migrations.reject do |migration| @@ -51,7 +79,7 @@ module Gitlab `git diff --name-only origin/HEAD...HEAD db/post_migrate db/migrate`.split("\n") end - def migrations_for_down + def migrations_for_down(database) versions_this_branch = migration_file_names_this_branch.map do |m_name| m_name.match(%r{^db/(post_)?migrate/(\d+)}) { |m| m.captures[1]&.to_i } end.to_set @@ -65,14 +93,21 @@ module Gitlab attr_reader :direction, :result_dir, :migrations - delegate :migration_context, to: :class + delegate :migration_context, :within_context_for_database, to: :class - def initialize(direction:, migrations:, result_dir:) + def initialize(direction:, database:, migrations:, legacy_mode: false) raise "Direction must be up or down" unless %i[up down].include?(direction) @direction = direction @migrations = migrations - @result_dir = result_dir + @result_dir = if legacy_mode + BASE_RESULT_DIR.join(direction.to_s) + else + BASE_RESULT_DIR.join(database.to_s, direction.to_s) + end + + @database = database + @legacy_mode = legacy_mode end def run @@ -86,14 +121,22 @@ module Gitlab instrumentation = Instrumentation.new(result_dir: result_dir) - sorted_migrations.each do |migration| - instrumentation.observe(version: migration.version, name: migration.name, connection: ActiveRecord::Migration.connection) do - ActiveRecord::Migrator.new(direction, migration_context.migrations, migration_context.schema_migration, migration.version).run + within_context_for_database(@database) do + sorted_migrations.each do |migration| + instrumentation.observe(version: migration.version, name: migration.name, connection: ActiveRecord::Migration.connection) do + ActiveRecord::Migrator.new(direction, migration_context.migrations, migration_context.schema_migration, migration.version).run + end end end ensure metadata_filename = File.join(result_dir, METADATA_FILENAME) - File.write(metadata_filename, { version: SCHEMA_VERSION }.to_json) + version = if @legacy_mode + 3 + else + SCHEMA_VERSION + end + + File.write(metadata_filename, { database: @database.to_s, version: version }.to_json) # We clear the cache here to mirror the cache clearing that happens at the end of `db:migrate` tasks # This clearing makes subsequent rake tasks in the same execution pick up database schema changes caused by diff --git a/lib/gitlab/database/migrations/test_background_runner.rb b/lib/gitlab/database/migrations/test_background_runner.rb index 6da2e098d43..65db330b1a6 100644 --- a/lib/gitlab/database/migrations/test_background_runner.rb +++ b/lib/gitlab/database/migrations/test_background_runner.rb @@ -5,7 +5,7 @@ module Gitlab module Migrations class TestBackgroundRunner < BaseBackgroundRunner def initialize(result_dir:) - super(result_dir: result_dir) + super(result_dir: result_dir, connection: ActiveRecord::Migration.connection) @job_coordinator = Gitlab::BackgroundMigration.coordinator_for_database(Gitlab::Database::MAIN_DATABASE_NAME) end diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index c27ae6a2c5d..46855ca1921 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -5,65 +5,73 @@ module Gitlab module Migrations class TestBatchedBackgroundRunner < BaseBackgroundRunner include Gitlab::Database::DynamicModelHelpers - attr_reader :connection def initialize(result_dir:, connection:) - super(result_dir: result_dir) + super(result_dir: result_dir, connection: connection) @connection = connection end def jobs_by_migration_name - Gitlab::Database::BackgroundMigration::BatchedMigration - .executable - .created_after(3.hours.ago) # Simple way to exclude migrations already running before migration testing - .to_h do |migration| - batching_strategy = migration.batch_class.new(connection: connection) - - smallest_batch_start = migration.next_min_value - - table_max_value = define_batchable_model(migration.table_name, connection: connection) - .maximum(migration.column_name) - - largest_batch_start = table_max_value - migration.batch_size - - # variance is the portion of the batch range that we shrink between variance * 0 and variance * 1 - # to pick actual batches to sample. - variance = largest_batch_start - smallest_batch_start - - batch_starts = uniform_fractions - .lazy # frac varies from 0 to 1, values in smallest_batch_start..largest_batch_start - .map { |frac| (variance * frac).to_i + smallest_batch_start } - - # Track previously run batches so that we stop sampling if a new batch would intersect an older one - completed_batches = [] - - jobs_to_sample = batch_starts - # Stop sampling if a batch would intersect a previous batch - .take_while { |start| completed_batches.none? { |batch| batch.cover?(start) } } - .map do |batch_start| - next_bounds = batching_strategy.next_batch( - migration.table_name, - migration.column_name, - batch_min_value: batch_start, - batch_size: migration.batch_size, - job_arguments: migration.job_arguments - ) - - batch_min, batch_max = next_bounds - - job = migration.create_batched_job!(batch_min, batch_max) - - completed_batches << (batch_min..batch_max) + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::BackgroundMigration::BatchedMigration + .executable + .created_after(3.hours.ago) # Simple way to exclude migrations already running before migration testing + .to_h do |migration| + batching_strategy = migration.batch_class.new(connection: connection) + + smallest_batch_start = migration.next_min_value + + table_max_value = define_batchable_model(migration.table_name, connection: connection) + .maximum(migration.column_name) + + largest_batch_start = table_max_value - migration.batch_size + + # variance is the portion of the batch range that we shrink between variance * 0 and variance * 1 + # to pick actual batches to sample. + variance = largest_batch_start - smallest_batch_start + + batch_starts = uniform_fractions + .lazy # frac varies from 0 to 1, values in smallest_batch_start..largest_batch_start + .map { |frac| (variance * frac).to_i + smallest_batch_start } + + # Track previously run batches so that we stop sampling if a new batch would intersect an older one + completed_batches = [] + + jobs_to_sample = batch_starts + # Stop sampling if a batch would intersect a previous batch + .take_while { |start| completed_batches.none? { |batch| batch.cover?(start) } } + .map do |batch_start| + # The current block is lazily evaluated as part of the jobs_to_sample enumerable + # so it executes after the enclosing using_connection block has already executed + # Therefore we need to re-associate with the explicit connection again + Gitlab::Database::SharedModel.using_connection(connection) do + next_bounds = batching_strategy.next_batch( + migration.table_name, + migration.column_name, + batch_min_value: batch_start, + batch_size: migration.batch_size, + job_arguments: migration.job_arguments + ) + + batch_min, batch_max = next_bounds + + job = migration.create_batched_job!(batch_min, batch_max) + + completed_batches << (batch_min..batch_max) + + job + end + end - job + [migration.job_class_name, jobs_to_sample] end - - [migration.job_class_name, jobs_to_sample] end end def run_job(job) - Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper.new(connection: connection).perform(job) + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper.new(connection: connection).perform(job) + end end def uniform_fractions diff --git a/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb b/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb index f45cf02ec9b..23a8dc0b44f 100644 --- a/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb +++ b/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb @@ -6,8 +6,6 @@ module Gitlab class ConvertTableToFirstListPartition UnableToPartition = Class.new(StandardError) - include Gitlab::Database::MigrationHelpers - SQL_STATEMENT_SEPARATOR = ";\n\n" attr_reader :partitioning_column, :table_name, :parent_table_name, :zero_partition_value @@ -175,9 +173,21 @@ module Gitlab def alter_sequence_statements(old_table:, new_table:) sequences_owned_by(old_table).map do |seq_info| seq_name, column_name = seq_info.values_at(:name, :column_name) - <<~SQL.chomp + + statement_parts = [] + + # If a different user owns the old table, the conversion process will fail to reassign the sequence + # ownership to the new parent table (as it will be owned by the current user). + # Force the old table to be owned by the current user in that case. + unless current_user_owns_table?(old_table) + statement_parts << set_current_user_owns_table_statement(old_table) + end + + statement_parts << <<~SQL.chomp ALTER SEQUENCE #{quote_table_name(seq_name)} OWNED BY #{quote_table_name(new_table)}.#{quote_column_name(column_name)} SQL + + statement_parts.join(SQL_STATEMENT_SEPARATOR) end end @@ -208,6 +218,23 @@ module Gitlab { name: name, column_name: column_name } end end + + def table_owner(table_name) + connection.select_value(<<~SQL, nil, [table_name]) + SELECT tableowner FROM pg_tables WHERE tablename = $1 + SQL + end + + def current_user_owns_table?(table_name) + current_user = connection.select_value('select current_user') + table_owner(table_name) == current_user + end + + def set_current_user_owns_table_statement(table_name) + <<~SQL.chomp + ALTER TABLE #{connection.quote_table_name(table_name)} OWNER TO CURRENT_USER + SQL + end end end end diff --git a/lib/gitlab/database/query_analyzers/base.rb b/lib/gitlab/database/query_analyzers/base.rb index 5f321ece962..9a52a4f6e23 100644 --- a/lib/gitlab/database/query_analyzers/base.rb +++ b/lib/gitlab/database/query_analyzers/base.rb @@ -8,7 +8,7 @@ module Gitlab QueryAnalyzerError = Class.new(Exception) # rubocop:disable Lint/InheritException def self.suppressed? - Thread.current[self.suppress_key] + Thread.current[self.suppress_key] || @suppress_in_rspec end def self.requires_tracking?(parsed) @@ -19,6 +19,20 @@ module Gitlab Thread.current[self.suppress_key] = value end + # The other suppress= method stores the + # value in Thread.current because it is + # meant to work in a multi-threaded puma + # environment but this does not work + # correctly in capybara tests where we + # suppress in the rspec runner context but + # this does not take effect in the puma + # thread. As such we just suppress + # globally in RSpec since we don't run + # different tests concurrently. + class << self + attr_writer :suppress_in_rspec + end + def self.with_suppressed(value = true, &blk) previous = self.suppressed? self.suppress = value diff --git a/lib/gitlab/database/reflection.rb b/lib/gitlab/database/reflection.rb index 33c965cb150..6c4e46728d4 100644 --- a/lib/gitlab/database/reflection.rb +++ b/lib/gitlab/database/reflection.rb @@ -124,7 +124,11 @@ module Gitlab # - https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-servers # - https://docs.microsoft.com/en-us/azure/postgresql/concepts-servers#managing-your-server # this database is present on both Flexible and Single server, so we should check the former first. - 'Azure Database for PostgreSQL - Single Server' => { statement: "SELECT datname FROM pg_database WHERE datname = 'azure_maintenance'" } + 'Azure Database for PostgreSQL - Single Server' => { statement: "SELECT datname FROM pg_database WHERE datname = 'azure_maintenance'" }, + # Based on + # - https://cloud.google.com/sql/docs/postgres/flags + # running a query to detect flag names that begin with 'alloydb + 'AlloyDB for PostgreSQL' => { statement: "SELECT name FROM pg_settings WHERE name LIKE 'alloydb%'" } }.each do |flavor, conditions| return flavor if connection.execute(conditions[:statement]).to_a.present? rescue ActiveRecord::StatementInvalid => e diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb index f96de13006f..2c7ca28942e 100644 --- a/lib/gitlab/database/schema_helpers.rb +++ b/lib/gitlab/database/schema_helpers.rb @@ -71,12 +71,17 @@ module Gitlab "#{type}_#{hashed_identifier}" end - def with_lock_retries(&block) - Gitlab::Database::WithLockRetries.new( + def with_lock_retries(*args, **kwargs, &block) + raise_on_exhaustion = !!kwargs.delete(:raise_on_exhaustion) + merged_args = { connection: connection, klass: self.class, - logger: Gitlab::BackgroundMigration::Logger - ).run(&block) + logger: Gitlab::BackgroundMigration::Logger, + allow_savepoints: true + }.merge(kwargs) + + Gitlab::Database::WithLockRetries.new(**merged_args) + .run(raise_on_exhaustion: raise_on_exhaustion, &block) end def assert_not_in_transaction_block(scope:) diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 084ce63e36a..d6f5e45c034 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -82,16 +82,6 @@ module Gitlab private - def expiration - return 1.day unless Feature.enabled?(:highlight_diffs_renewable_expiration, diffable.project) - - if Feature.enabled?(:highlight_diffs_short_renewable_expiration, diffable.project) - EXPIRATION - else - 8.hours - end - end - def set_highlighted_diff_lines(diff_file, content) diff_file.highlighted_diff_lines = content.map do |line| Gitlab::Diff::Line.safe_init_from_hash(line) @@ -147,7 +137,7 @@ module Gitlab end # HSETs have to have their expiration date manually updated - pipeline.expire(key, expiration) + pipeline.expire(key, EXPIRATION) end record_memory_usage(fetch_memory_usage(redis, key)) @@ -197,14 +187,12 @@ module Gitlab return {} unless file_paths.any? results = [] - cache_key = key - highlight_diffs_renewable_expiration_enabled = Feature.enabled?(:highlight_diffs_renewable_expiration, diffable.project) - expiration_period = expiration + cache_key = key # Moving out redis calls for feature flags out of redis.pipelined Gitlab::Redis::Cache.with do |redis| redis.pipelined do |pipeline| results = pipeline.hmget(cache_key, file_paths) - pipeline.expire(key, expiration_period) if highlight_diffs_renewable_expiration_enabled + pipeline.expire(key, EXPIRATION) end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 7b31dd9926b..434893eab82 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -36,8 +36,14 @@ module Gitlab validate_permission!(:create_issue) + result = create_issue + issue = result[:issue] + + # issue won't be present only on unrecoverable errors + raise InvalidIssueError, result.errors.join(', ') if result.error? && issue.blank? + verify_record!( - record: create_issue, + record: issue, invalid_exception: InvalidIssueError, record_name: 'issue') end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 8e2c7559bc1..06365296a76 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -91,7 +91,7 @@ module Gitlab end def create_issue! - @issue = ::Issues::CreateService.new( + result = ::Issues::CreateService.new( project: project, current_user: User.support_bot, params: { @@ -106,7 +106,9 @@ module Gitlab spam_params: nil ).execute - raise InvalidIssueError unless @issue.persisted? + raise InvalidIssueError if result.error? + + @issue = result[:issue] begin ::Issue::Email.create!(issue: @issue, email_message_id: mail.message_id) diff --git a/lib/gitlab/email/message/in_product_marketing/trial.rb b/lib/gitlab/email/message/in_product_marketing/trial.rb index 11a799886ab..720262816b4 100644 --- a/lib/gitlab/email/message/in_product_marketing/trial.rb +++ b/lib/gitlab/email/message/in_product_marketing/trial.rb @@ -42,11 +42,11 @@ module Gitlab [ s_("InProductMarketing|GitLab's premium tiers are designed to make you, your team and your application more efficient and more secure with features including but not limited to:"), list([ - s_('InProductMarketing|%{strong_start}Company wide portfolio management%{strong_end} — including multi-level epics, scoped labels').html_safe % strong_options, - s_('InProductMarketing|%{strong_start}Multiple approval roles%{strong_end} — including code owners and required merge approvals').html_safe % strong_options, - s_('InProductMarketing|%{strong_start}Advanced application security%{strong_end} — including SAST, DAST scanning, FUZZ testing, dependency scanning, license compliance, secrete detection').html_safe % strong_options, - s_('InProductMarketing|%{strong_start}Executive level insights%{strong_end} — including reporting on productivity, tasks by type, days to completion, value stream').html_safe % strong_options - ]) + s_('InProductMarketing|%{strong_start}Company wide portfolio management%{strong_end} — including multi-level epics, scoped labels').html_safe % strong_options, + s_('InProductMarketing|%{strong_start}Multiple approval roles%{strong_end} — including code owners and required merge approvals').html_safe % strong_options, + s_('InProductMarketing|%{strong_start}Advanced application security%{strong_end} — including SAST, DAST scanning, FUZZ testing, dependency scanning, license compliance, secrete detection').html_safe % strong_options, + s_('InProductMarketing|%{strong_start}Executive level insights%{strong_end} — including reporting on productivity, tasks by type, days to completion, value stream').html_safe % strong_options + ]) ].join("\n"), s_('InProductMarketing|GitLab provides static application security testing (SAST), dynamic application security testing (DAST), container scanning, and dependency scanning to help you deliver secure applications along with license compliance.'), s_('InProductMarketing|By enabling code owners and required merge approvals the right person will review the right MR. This is a win-win: cleaner code and a more efficient review process.') diff --git a/lib/gitlab/email/message/in_product_marketing/verify.rb b/lib/gitlab/email/message/in_product_marketing/verify.rb index d2a78b53e1f..3982a8b87fd 100644 --- a/lib/gitlab/email/message/in_product_marketing/verify.rb +++ b/lib/gitlab/email/message/in_product_marketing/verify.rb @@ -49,10 +49,10 @@ module Gitlab [ nil, list([ - s_('InProductMarketing|Start by %{performance_link}').html_safe % { performance_link: performance_link }, - s_('InProductMarketing|Move on to easily creating a Pages website %{ci_template_link}').html_safe % { ci_template_link: ci_template_link }, - s_('InProductMarketing|And finally %{deploy_link} a Python application.').html_safe % { deploy_link: deploy_link } - ]), + s_('InProductMarketing|Start by %{performance_link}').html_safe % { performance_link: performance_link }, + s_('InProductMarketing|Move on to easily creating a Pages website %{ci_template_link}').html_safe % { ci_template_link: ci_template_link }, + s_('InProductMarketing|And finally %{deploy_link} a Python application.').html_safe % { deploy_link: deploy_link } + ]), nil ][series] end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 34c674c3003..b1fd35184ac 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -73,8 +73,6 @@ module Gitlab # This method escapes unsupported UTF-8 characters instead of deleting them def encode_utf8_with_escaping!(message) - return encode!(message) if Feature.disabled?(:escape_gitaly_refs) - message = force_encode_utf8(message) return message if message.valid_encoding? diff --git a/lib/gitlab/environment.rb b/lib/gitlab/environment.rb index b1a9603d3a5..3c6ed696b9d 100644 --- a/lib/gitlab/environment.rb +++ b/lib/gitlab/environment.rb @@ -5,5 +5,9 @@ module Gitlab def self.hostname @hostname ||= ENV['HOSTNAME'] || Socket.gethostname end + + def self.qa_user_agent + ENV['GITLAB_QA_USER_AGENT'] + end end end diff --git a/lib/gitlab/event_store.rb b/lib/gitlab/event_store.rb index b45970cb45a..023c8ace4d9 100644 --- a/lib/gitlab/event_store.rb +++ b/lib/gitlab/event_store.rb @@ -43,9 +43,18 @@ module Gitlab store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectPathChangedEvent store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectArchivedEvent store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectTransferedEvent + store.subscribe ::Pages::InvalidateDomainCacheWorker, + to: ::Projects::ProjectAttributesChangedEvent, + if: -> (event) { event.pages_related? } + store.subscribe ::Pages::InvalidateDomainCacheWorker, + to: ::Projects::ProjectFeaturesChangedEvent, + if: -> (event) { event.pages_related? } store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Groups::GroupTransferedEvent store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Groups::GroupPathChangedEvent store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Groups::GroupDeletedEvent + store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::PagesDomains::PagesDomainDeletedEvent + store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::PagesDomains::PagesDomainUpdatedEvent + store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::PagesDomains::PagesDomainCreatedEvent store.subscribe ::MergeRequests::CreateApprovalEventWorker, to: ::MergeRequests::ApprovedEvent store.subscribe ::MergeRequests::CreateApprovalNoteWorker, to: ::MergeRequests::ApprovedEvent diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 75d07a36dcd..0b18a337707 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -118,7 +118,7 @@ module Gitlab # Returns true if the key for this lease is set. def exists? Gitlab::Redis::SharedState.with do |redis| - redis.exists(@redis_shared_state_key) + redis.exists?(@redis_shared_state_key) # rubocop:disable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb deleted file mode 100644 index 142d0e55593..00000000000 --- a/lib/gitlab/experimentation.rb +++ /dev/null @@ -1,110 +0,0 @@ -# frozen_string_literal: true - -# == Experimentation -# -# Utility module for A/B testing experimental features. Define your experiments in the `EXPERIMENTS` constant. -# Experiment options: -# - tracking_category (optional, used to set the category when tracking an experiment event) -# - rollout_strategy: default is `:cookie` based rollout. We may also set it to `:user` based rollout -# -# The experiment is controlled by a Feature Flag (https://docs.gitlab.com/ee/development/feature_flags/controls.html), -# which is named "#{experiment_key}_experiment_percentage" and *must* be set with a percentage and not be used for other purposes. -# -# To enable the experiment for 10% of the time: -# -# chatops: `/chatops run feature set experiment_key_experiment_percentage 10 --random` -# console: `Feature.enable_percentage_of_time(:experiment_key_experiment_percentage, 10)` -# -# To disable the experiment: -# -# chatops: `/chatops run feature delete experiment_key_experiment_percentage` -# console: `Feature.remove(:experiment_key_experiment_percentage)` -# -# To check the current rollout percentage: -# -# chatops: `/chatops run feature get experiment_key_experiment_percentage` -# console: `Feature.get(:experiment_key_experiment_percentage).percentage_of_time_value` -# - -# TODO: see https://gitlab.com/gitlab-org/gitlab/-/issues/217490 -module Gitlab - module Experimentation - EXPERIMENTS = { - }.freeze - - class << self - def get_experiment(experiment_key) - return unless EXPERIMENTS.key?(experiment_key) - - ::Gitlab::Experimentation::Experiment.new(experiment_key, **EXPERIMENTS[experiment_key]) - end - - def active?(experiment_key) - experiment = get_experiment(experiment_key) - return false unless experiment - - experiment.active? - end - - def in_experiment_group?(experiment_key, subject:) - return false if subject.blank? - return false unless active?(experiment_key) - - log_invalid_rollout(experiment_key, subject) - - experiment = get_experiment(experiment_key) - return false unless experiment - - experiment.enabled_for_index?(index_for_subject(experiment, subject)) - end - - def rollout_strategy(experiment_key) - experiment = get_experiment(experiment_key) - return unless experiment - - experiment.rollout_strategy - end - - def log_invalid_rollout(experiment_key, subject) - return if valid_subject_for_rollout_strategy?(experiment_key, subject) - - logger = Gitlab::ExperimentationLogger.build - logger.warn message: 'Subject must conform to the rollout strategy', - experiment_key: experiment_key, - subject: subject.class.to_s, - rollout_strategy: rollout_strategy(experiment_key) - end - - def valid_subject_for_rollout_strategy?(experiment_key, subject) - case rollout_strategy(experiment_key) - when :user - subject.is_a?(User) - when :group - subject.is_a?(Group) - when :cookie - subject.nil? || subject.is_a?(String) - else - false - end - end - - private - - def index_for_subject(experiment, subject) - index = Zlib.crc32("#{experiment.key}#{subject_id(subject)}") - - index % 100 - end - - def subject_id(subject) - if subject.respond_to?(:to_global_id) - subject.to_global_id.to_s - elsif subject.respond_to?(:to_s) - subject.to_s - else - raise ArgumentError, 'Subject must respond to `to_global_id` or `to_s`' - end - end - end - end -end diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb deleted file mode 100644 index b09d67b8d5f..00000000000 --- a/lib/gitlab/experimentation/controller_concern.rb +++ /dev/null @@ -1,156 +0,0 @@ -# frozen_string_literal: true - -require 'zlib' - -# Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent. -# Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name, subject: nil)` method -# to controllers and views. It returns true when the experiment is enabled and the user is selected as part -# of the experimental group. -# -module Gitlab - module Experimentation - module ControllerConcern - include ::Gitlab::Experimentation::GroupTypes - include Gitlab::Tracking::Helpers - extend ActiveSupport::Concern - - included do - before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? - helper_method :experiment_enabled?, :experiment_tracking_category_and_group, :record_experiment_group - end - - def set_experimentation_subject_id_cookie - if Gitlab.com? - return if cookies[:experimentation_subject_id].present? - - cookies.permanent.signed[:experimentation_subject_id] = { - value: SecureRandom.uuid, - secure: ::Gitlab.config.gitlab.https, - httponly: true - } - else - # We set the cookie before, although experiments are not conducted on self managed instances. - cookies.delete(:experimentation_subject_id) - end - end - - def push_frontend_experiment(experiment_key, subject: nil) - var_name = experiment_key.to_s.camelize(:lower) - - enabled = experiment_enabled?(experiment_key, subject: subject) - - gon.push({ experiments: { var_name => enabled } }, true) - end - - def experiment_enabled?(experiment_key, subject: nil) - return true if forced_enabled?(experiment_key) - return false if dnt_enabled? - - Experimentation.log_invalid_rollout(experiment_key, subject) - - subject ||= experimentation_subject_id - - Experimentation.in_experiment_group?(experiment_key, subject: subject) - end - - def track_experiment_event(experiment_key, action, value = nil, subject: nil) - return if dnt_enabled? - - track_experiment_event_for(experiment_key, action, value, subject: subject) do |tracking_data| - ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data.merge!(user: current_user)) - end - end - - def frontend_experimentation_tracking_data(experiment_key, action, value = nil, subject: nil) - return if dnt_enabled? - - track_experiment_event_for(experiment_key, action, value, subject: subject) do |tracking_data| - gon.push(tracking_data: tracking_data) - end - end - - def record_experiment_user(experiment_key, context = {}) - return if dnt_enabled? - return unless Experimentation.active?(experiment_key) && current_user - - subject = Experimentation.rollout_strategy(experiment_key) == :cookie ? nil : current_user - - ::Experiment.add_user(experiment_key, tracking_group(experiment_key, nil, subject: subject), current_user, context) - end - - def record_experiment_group(experiment_key, group) - return if dnt_enabled? - return unless Experimentation.active?(experiment_key) && group - - variant_subject = Experimentation.rollout_strategy(experiment_key) == :cookie ? nil : group - variant = tracking_group(experiment_key, nil, subject: variant_subject) - - ::Experiment.add_group(experiment_key, group: group, variant: variant) - end - - def record_experiment_conversion_event(experiment_key, context = {}) - return if dnt_enabled? - return unless current_user - return unless Experimentation.active?(experiment_key) - - ::Experiment.record_conversion_event(experiment_key, current_user, context) - end - - def experiment_tracking_category_and_group(experiment_key, subject: nil) - "#{tracking_category(experiment_key)}:#{tracking_group(experiment_key, '_group', subject: subject)}" - end - - private - - def experimentation_subject_id - cookies.signed[:experimentation_subject_id] - end - - def track_experiment_event_for(experiment_key, action, value, subject: nil) - return unless Experimentation.active?(experiment_key) - - yield experimentation_tracking_data(experiment_key, action, value, subject: subject) - end - - def experimentation_tracking_data(experiment_key, action, value, subject: nil) - { - category: tracking_category(experiment_key), - action: action, - property: tracking_group(experiment_key, "_group", subject: subject), - label: tracking_label(subject), - value: value - }.compact - end - - def tracking_category(experiment_key) - Experimentation.get_experiment(experiment_key).tracking_category - end - - def tracking_group(experiment_key, suffix = nil, subject: nil) - return unless Experimentation.active?(experiment_key) - - subject ||= experimentation_subject_id - group = experiment_enabled?(experiment_key, subject: subject) ? GROUP_EXPERIMENTAL : GROUP_CONTROL - - suffix ? "#{group}#{suffix}" : group - end - - def forced_enabled?(experiment_key) - return true if params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s - return false if cookies[:force_experiment].blank? - - cookies[:force_experiment].to_s.split(',').any? { |experiment| experiment.strip == experiment_key.to_s } - end - - def tracking_label(subject = nil) - return experimentation_subject_id if subject.blank? - - if subject.respond_to?(:to_global_id) - Digest::SHA256.hexdigest(subject.to_global_id.to_s) - else - Digest::SHA256.hexdigest(subject.to_s) - end - end - end - end -end diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb deleted file mode 100644 index 0c7091d19e3..00000000000 --- a/lib/gitlab/experimentation/experiment.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Experimentation - class Experiment - FEATURE_FLAG_SUFFIX = "_experiment_percentage" - - attr_reader :key, :tracking_category, :rollout_strategy - - def initialize(key, **params) - @key = key - @tracking_category = params[:tracking_category] - @rollout_strategy = params[:rollout_strategy] || :cookie - end - - def active? - # TODO: just touch a feature flag - # Temporary change, we will change `experiment_percentage` in future to `Feature.enabled? - Feature.enabled?(feature_flag_name, type: :experiment) - - ::Gitlab.com? && experiment_percentage > 0 - end - - def enabled_for_index?(index) - return false if index.blank? - - index <= experiment_percentage - end - - private - - def experiment_percentage - feature_flag.percentage_of_time_value - end - - def feature_flag - Feature.get(feature_flag_name) # rubocop:disable Gitlab/AvoidFeatureGet - end - - def feature_flag_name - :"#{key}#{FEATURE_FLAG_SUFFIX}" - end - end - end -end diff --git a/lib/gitlab/experimentation_logger.rb b/lib/gitlab/experimentation_logger.rb deleted file mode 100644 index ba1b60d6b4c..00000000000 --- a/lib/gitlab/experimentation_logger.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class ExperimentationLogger < ::Gitlab::JsonLogger - def self.file_name_noext - 'experimentation_json' - end - end -end diff --git a/lib/gitlab/git/declared_license.rb b/lib/gitlab/git/declared_license.rb new file mode 100644 index 00000000000..bc12b1918ea --- /dev/null +++ b/lib/gitlab/git/declared_license.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Git + # DeclaredLicense is the software license declared in a LICENSE or COPYING + # file in the git repository. + class DeclaredLicense + # SPDX Identifier for the license + attr_reader :key + + # Full name of the license + attr_reader :name + + # Nickname of the license (optional, a shorter user-friendly name) + attr_reader :nickname + + # Filename of the file containing license + attr_accessor :path + + # URL that points to the LICENSE + attr_reader :url + + def initialize(key: nil, name: nil, nickname: nil, url: nil, path: nil) + @key = key + @name = name + @nickname = nickname + @url = url + @path = path + end + + def ==(other) + return unless other.is_a?(self.class) + + key == other.key + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index f1cd75258be..9bbe17dcad1 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -783,10 +783,31 @@ module Gitlab end end - def license_short_name + def license(from_gitaly) wrapped_gitaly_errors do - gitaly_repository_client.license_short_name + response = gitaly_repository_client.find_license + + break nil if response.license_short_name.empty? + + if from_gitaly + break ::Gitlab::Git::DeclaredLicense.new(key: response.license_short_name, + name: response.license_name, + nickname: response.license_nickname.presence, + url: response.license_url.presence, + path: response.license_path) + end + + licensee_object = Licensee::License.new(response.license_short_name) + + break nil if licensee_object.name.blank? + + licensee_object.meta.nickname = "LICENSE" if licensee_object.key == "other" + + licensee_object end + rescue Licensee::InvalidLicense => e + Gitlab::ErrorTracking.track_exception(e) + nil end def fetch_source_branch!(source_repository, source_branch, local_ref) @@ -1008,8 +1029,8 @@ module Gitlab @praefect_info_client ||= Gitlab::GitalyClient::PraefectInfoService.new(self) end - def branch_names_contains_sha(sha) - gitaly_ref_client.branch_names_contains_sha(sha) + def branch_names_contains_sha(sha, limit: 0) + gitaly_ref_client.branch_names_contains_sha(sha, limit: limit) end def tag_names_contains_sha(sha, limit: 0) diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb deleted file mode 100644 index 2228fcb886e..00000000000 --- a/lib/gitlab/git/wiki.rb +++ /dev/null @@ -1,131 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Git - class Wiki - include Gitlab::Git::WrapsGitalyErrors - - DuplicatePageError = Class.new(StandardError) - - DEFAULT_PAGINATION = Kaminari.config.default_per_page - - CommitDetails = Struct.new(:user_id, :username, :name, :email, :message) do - def to_h - { user_id: user_id, username: username, name: name, email: email, message: message } - end - end - - # GollumSlug inlines just enough knowledge from Gollum::Page to generate a - # slug, which is used when previewing pages that haven't been persisted - class GollumSlug - class << self - def cname(name, char_white_sub = '-', char_other_sub = '-') - if name.respond_to?(:gsub) - name.gsub(/\s/, char_white_sub).gsub(/[<>+]/, char_other_sub) - else - '' - end - end - - def format_to_ext(format) - format == :markdown ? "md" : format.to_s - end - - def canonicalize_filename(filename) - ::File.basename(filename, ::File.extname(filename)).tr('-', ' ') - end - - def generate(title, format) - ext = format_to_ext(format.to_sym) - name = cname(title) + '.' + ext - canonical_name = canonicalize_filename(name) - - path = - if name.include?('/') - name.sub(%r{/[^/]+$}, '/') - else - '' - end - - path + cname(canonical_name, '-', '-') - end - end - end - - attr_reader :repository - - # TODO remove argument when issue - # https://gitlab.com/gitlab-org/gitlab/-/issues/329190 - # is closed. - def self.default_ref(container = nil) - Gitlab::DefaultBranch.value(object: container) - end - - # Initialize with a Gitlab::Git::Repository instance - def initialize(repository) - @repository = repository - end - - def repository_exists? - @repository.exists? - end - - def list_pages(limit: 0, sort: nil, direction_desc: false, load_content: false) - wrapped_gitaly_errors do - gitaly_list_pages( - limit: limit, - sort: sort, - direction_desc: direction_desc, - load_content: load_content - ) - end - end - - def page(title:, version: nil, dir: nil, load_content: true) - wrapped_gitaly_errors do - gitaly_find_page(title: title, version: version, dir: dir, load_content: load_content) - end - end - - def count_page_versions(page_path) - @repository.count_commits(ref: 'HEAD', path: page_path) - end - - def preview_slug(title, format) - GollumSlug.generate(title, format) - end - - private - - def gitaly_wiki_client - @gitaly_wiki_client ||= Gitlab::GitalyClient::WikiService.new(@repository) - end - - def gitaly_find_page(title:, version: nil, dir: nil, load_content: true) - return unless title.present? - - wiki_page, version = gitaly_wiki_client.find_page(title: title, version: version, dir: dir, load_content: load_content) - return unless wiki_page - - Gitlab::Git::WikiPage.from_gitaly_wiki_page(wiki_page, version) - rescue GRPC::InvalidArgument - nil - end - - def gitaly_list_pages(limit: 0, sort: nil, direction_desc: false, load_content: false) - params = { limit: limit, sort: sort, direction_desc: direction_desc } - - gitaly_pages = - if load_content - gitaly_wiki_client.load_all_pages(**params) - else - gitaly_wiki_client.list_all_pages(**params) - end - - gitaly_pages.map do |wiki_page, version| - Gitlab::Git::WikiPage.from_gitaly_wiki_page(wiki_page, version) - end - end - end - end -end diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb index 57b7e7d53dd..26d15daf093 100644 --- a/lib/gitlab/git/wiki_page.rb +++ b/lib/gitlab/git/wiki_page.rb @@ -5,22 +5,6 @@ module Gitlab class WikiPage attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :historical, :formatted_data - class << self - # Abstracts away Gitlab::GitalyClient::WikiPage - def from_gitaly_wiki_page(gitaly_page, version) - new( - url_path: gitaly_page.url_path, - title: gitaly_page.title, - format: gitaly_page.format, - path: gitaly_page.path, - raw_data: gitaly_page.raw_data, - name: gitaly_page.name, - historical: gitaly_page.historical?, - version: version - ) - end - end - def initialize(hash) @url_path = hash[:url_path] @title = hash[:title] @@ -41,6 +25,11 @@ module Gitlab @text_data = @raw_data && Gitlab::EncodingHelper.encode!(@raw_data.dup) end + + def raw_data=(data) + @raw_data = data + @text_data = @raw_data && Gitlab::EncodingHelper.encode!(@raw_data.dup) + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 1c5ad650678..9a3f5fb844b 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -394,7 +394,7 @@ module Gitlab elsif user user.can?(:read_project, project) elsif ci? - true # allow CI (build without a user) for backwards compatibility + false end || Guest.can?(:read_project, project) end @@ -445,9 +445,6 @@ module Gitlab nil when Key actor.user - when :ci - Gitlab::AppJsonLogger.info(message: 'Actor was :ci', project_id: project.id) - nil end end end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index bb6bc3121bd..d2b702f3a6d 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -270,15 +270,13 @@ module Gitlab end def consume_find_local_branches_response(response) - if Feature.enabled?(:gitaly_simplify_find_local_branches_response, type: :undefined) - response.flat_map do |message| + response.flat_map do |message| + if message.local_branches.present? message.local_branches.map do |branch| target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) Gitlab::Git::Branch.new(@repository, branch.name, branch.target_commit.id, target_commit) end - end - else - response.flat_map do |message| + else message.branches.map do |gitaly_branch| Gitlab::Git::Branch.new( @repository, diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 04d6f92e8d8..f11437552e1 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -283,12 +283,10 @@ module Gitlab response.path.presence end - def license_short_name + def find_license request = Gitaly::FindLicenseRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :find_license, request, timeout: GitalyClient.fast_timeout) - - response.license_short_name.presence + GitalyClient.call(@storage, :repository_service, :find_license, request, timeout: GitalyClient.medium_timeout) end def calculate_checksum diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb deleted file mode 100644 index ca839b232cf..00000000000 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ /dev/null @@ -1,169 +0,0 @@ -# frozen_string_literal: true - -require 'stringio' - -module Gitlab - module GitalyClient - class WikiService - include Gitlab::EncodingHelper - - MAX_MSG_SIZE = 128.kilobytes.freeze - - def initialize(repository) - @gitaly_repo = repository.gitaly_repository - @repository = repository - end - - def write_page(name, format, content, commit_details) - request = Gitaly::WikiWritePageRequest.new( - repository: @gitaly_repo, - name: encode_binary(name), - format: format.to_s, - commit_details: gitaly_commit_details(commit_details) - ) - - strio = binary_io(content) - - enum = Enumerator.new do |y| - until strio.eof? - request.content = strio.read(MAX_MSG_SIZE) - - y.yield request - - request = Gitaly::WikiWritePageRequest.new - end - end - - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_write_page, enum, timeout: GitalyClient.medium_timeout) - if error = response.duplicate_error.presence - raise Gitlab::Git::Wiki::DuplicatePageError, error - end - end - - def update_page(page_path, title, format, content, commit_details) - request = Gitaly::WikiUpdatePageRequest.new( - repository: @gitaly_repo, - page_path: encode_binary(page_path), - title: encode_binary(title), - format: format.to_s, - commit_details: gitaly_commit_details(commit_details) - ) - - strio = binary_io(content) - - enum = Enumerator.new do |y| - until strio.eof? - request.content = strio.read(MAX_MSG_SIZE) - - y.yield request - - request = Gitaly::WikiUpdatePageRequest.new - end - end - - GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum, timeout: GitalyClient.medium_timeout) - end - - def find_page(title:, version: nil, dir: nil, load_content: true) - request = Gitaly::WikiFindPageRequest.new( - repository: @gitaly_repo, - title: encode_binary(title), - revision: encode_binary(version), - directory: encode_binary(dir), - skip_content: !load_content - ) - - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request, timeout: GitalyClient.fast_timeout) - - wiki_page_from_iterator(response) - end - - def list_all_pages(limit: 0, sort: nil, direction_desc: false) - sort_value = Gitaly::WikiListPagesRequest::SortBy.resolve(sort.to_s.upcase.to_sym) - - params = { repository: @gitaly_repo, limit: limit, direction_desc: direction_desc } - params[:sort] = sort_value if sort_value - - request = Gitaly::WikiListPagesRequest.new(params) - stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_list_pages, request, timeout: GitalyClient.medium_timeout) - stream.each_with_object([]) do |message, pages| - page = message.page - - next unless page - - wiki_page = GitalyClient::WikiPage.new(page.to_h) - version = new_wiki_page_version(page.version) - - pages << [wiki_page, version] - end - end - - def load_all_pages(limit: 0, sort: nil, direction_desc: false) - sort_value = Gitaly::WikiGetAllPagesRequest::SortBy.resolve(sort.to_s.upcase.to_sym) - - params = { repository: @gitaly_repo, limit: limit, direction_desc: direction_desc } - params[:sort] = sort_value if sort_value - - request = Gitaly::WikiGetAllPagesRequest.new(params) - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_all_pages, request, timeout: GitalyClient.medium_timeout) - - pages = [] - - loop do - page, version = wiki_page_from_iterator(response) { |message| message.end_of_page } - - break unless page && version - - pages << [page, version] - end - - pages - end - - private - - # If a block is given and the yielded value is truthy, iteration will be - # stopped early at that point; else the iterator is consumed entirely. - # The iterator is traversed with `next` to allow resuming the iteration. - def wiki_page_from_iterator(iterator) - wiki_page = version = nil - - while message = iterator.next - break if block_given? && yield(message) - - page = message.page - next unless page - - if wiki_page - wiki_page.raw_data << page.raw_data - else - wiki_page = GitalyClient::WikiPage.new(page.to_h) - - version = new_wiki_page_version(page.version) - end - end - - [wiki_page, version] - rescue StopIteration - [wiki_page, version] - end - - def new_wiki_page_version(version) - Gitlab::Git::WikiPageVersion.new( - Gitlab::Git::Commit.decorate(@repository, version.commit), - version.format - ) - end - - def gitaly_commit_details(commit_details) - Gitaly::WikiCommitDetails.new( - user_id: commit_details.user_id, - user_name: encode_binary(commit_details.username), - name: encode_binary(commit_details.name), - email: encode_binary(commit_details.email), - message: encode_binary(commit_details.message) - ) - end - end - end -end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 6cff15a204f..0f89a7b6575 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -69,7 +69,7 @@ module Gitlab # # username - The username of the user. def user(username) - with_rate_limit { octokit.user(username) } + with_rate_limit { octokit.user(username).to_h } end def pull_request_reviews(repo_name, iid) @@ -88,7 +88,7 @@ module Gitlab end def pull_request(repo_name, iid) - with_rate_limit { octokit.pull_request(repo_name, iid) } + with_rate_limit { octokit.pull_request(repo_name, iid).to_h } end def labels(*args) @@ -108,7 +108,7 @@ module Gitlab end def branch_protection(repo_name, branch_name) - with_rate_limit { octokit.branch_protection(repo_name, branch_name) } + with_rate_limit { octokit.branch_protection(repo_name, branch_name).to_h } end # Fetches data from the GitHub API and yields a Page object for every page @@ -150,7 +150,7 @@ module Gitlab each_page(method, *args) do |page| page.objects.each do |object| - yield object + yield object.to_h end end end @@ -183,7 +183,7 @@ module Gitlab end def search_query(str:, type:, include_collaborations: true, include_orgs: true) - query = "#{str} in:#{type} is:public,private user:#{octokit.user.login}" + query = "#{str} in:#{type} is:public,private user:#{octokit.user.to_h[:login]}" query = [query, collaborations_subquery].join(' ') if include_collaborations query = [query, organizations_subquery].join(' ') if include_orgs @@ -274,13 +274,13 @@ module Gitlab def collaborations_subquery each_object(:repos, nil, { affiliation: 'collaborator' }) - .map { |repo| "repo:#{repo.full_name}" } + .map { |repo| "repo:#{repo[:full_name]}" } .join(' ') end def organizations_subquery each_object(:organizations) - .map { |org| "org:#{org.login}" } + .map { |org| "org:#{org[:login]}" } .join(' ') end diff --git a/lib/gitlab/github_import/exceptions.rb b/lib/gitlab/github_import/exceptions.rb new file mode 100644 index 00000000000..3a36b64a11b --- /dev/null +++ b/lib/gitlab/github_import/exceptions.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Exceptions + # Sometimes it's not clear which of not implemented interfaces caused this error. + # We need custom exception to be able to add text that gives extra context. + NotImplementedError = Class.new(StandardError) + end + end +end diff --git a/lib/gitlab/github_import/importer/attachments/base_importer.rb b/lib/gitlab/github_import/importer/attachments/base_importer.rb new file mode 100644 index 00000000000..eaff99aed43 --- /dev/null +++ b/lib/gitlab/github_import/importer/attachments/base_importer.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Attachments + class BaseImporter + include ParallelScheduling + + BATCH_SIZE = 100 + + # The method that will be called for traversing through all the objects to + # import, yielding them to the supplied block. + def each_object_to_import + collection.each_batch(of: BATCH_SIZE, column: ordering_column) do |batch| + batch.each do |record| + next if already_imported?(record) + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + yield record + + # We mark the object as imported immediately so we don't end up + # scheduling it multiple times. + mark_as_imported(record) + end + end + end + + def representation_class + Representation::NoteText + end + + def importer_class + NoteAttachmentsImporter + end + + private + + def collection + raise Gitlab::GithubImport::Exceptions::NotImplementedError, '#collection' + end + + def ordering_column + :id + end + + def object_representation(object) + representation_class.from_db_record(object) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/attachments/issues_importer.rb b/lib/gitlab/github_import/importer/attachments/issues_importer.rb new file mode 100644 index 00000000000..090bfb4a098 --- /dev/null +++ b/lib/gitlab/github_import/importer/attachments/issues_importer.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Attachments + class IssuesImporter < ::Gitlab::GithubImport::Importer::Attachments::BaseImporter + def sidekiq_worker_class + ::Gitlab::GithubImport::Attachments::ImportIssueWorker + end + + def collection_method + :issue_attachments + end + + def object_type + :issue_attachment + end + + def id_for_already_imported_cache(issue) + issue.id + end + + private + + def collection + project.issues.select(:id, :description) + end + + def ordering_column + :iid + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb b/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb new file mode 100644 index 00000000000..f41071b1785 --- /dev/null +++ b/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Attachments + class MergeRequestsImporter < ::Gitlab::GithubImport::Importer::Attachments::BaseImporter + def sidekiq_worker_class + ::Gitlab::GithubImport::Attachments::ImportMergeRequestWorker + end + + def collection_method + :merge_request_attachments + end + + def object_type + :merge_request_attachment + end + + def id_for_already_imported_cache(merge_request) + merge_request.id + end + + private + + def collection + project.merge_requests.select(:id, :description) + end + + def ordering_column + :iid + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/attachments/notes_importer.rb b/lib/gitlab/github_import/importer/attachments/notes_importer.rb new file mode 100644 index 00000000000..aa38a7a3a3f --- /dev/null +++ b/lib/gitlab/github_import/importer/attachments/notes_importer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Attachments + class NotesImporter < ::Gitlab::GithubImport::Importer::Attachments::BaseImporter + def sidekiq_worker_class + ::Gitlab::GithubImport::Attachments::ImportNoteWorker + end + + def collection_method + :note_attachments + end + + def object_type + :note_attachment + end + + def id_for_already_imported_cache(note) + note.id + end + + private + + # TODO: exclude :system, :noteable_type from select after removing override Note#note method + # https://gitlab.com/gitlab-org/gitlab/-/issues/369923 + def collection + project.notes.user.select(:id, :note, :system, :noteable_type) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/attachments/releases_importer.rb b/lib/gitlab/github_import/importer/attachments/releases_importer.rb new file mode 100644 index 00000000000..feaa69eff71 --- /dev/null +++ b/lib/gitlab/github_import/importer/attachments/releases_importer.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Attachments + class ReleasesImporter < ::Gitlab::GithubImport::Importer::Attachments::BaseImporter + def sidekiq_worker_class + ::Gitlab::GithubImport::Attachments::ImportReleaseWorker + end + + def collection_method + :release_attachments + end + + def object_type + :release_attachment + end + + def id_for_already_imported_cache(release) + release.id + end + + private + + def collection + project.releases.select(:id, :description) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/diff_notes_importer.rb b/lib/gitlab/github_import/importer/diff_notes_importer.rb index 49cbc8f7a42..92f26692a05 100644 --- a/lib/gitlab/github_import/importer/diff_notes_importer.rb +++ b/lib/gitlab/github_import/importer/diff_notes_importer.rb @@ -27,7 +27,7 @@ module Gitlab end def id_for_already_imported_cache(note) - note.id + note[:id] end end end diff --git a/lib/gitlab/github_import/importer/issue_events_importer.rb b/lib/gitlab/github_import/importer/issue_events_importer.rb index 71dd99f91f9..a1c706c5d78 100644 --- a/lib/gitlab/github_import/importer/issue_events_importer.rb +++ b/lib/gitlab/github_import/importer/issue_events_importer.rb @@ -27,7 +27,7 @@ module Gitlab end def id_for_already_imported_cache(event) - event.id + event[:id] end end end diff --git a/lib/gitlab/github_import/importer/issues_importer.rb b/lib/gitlab/github_import/importer/issues_importer.rb index 21d9ce8cd2d..3d6f15fc2bc 100644 --- a/lib/gitlab/github_import/importer/issues_importer.rb +++ b/lib/gitlab/github_import/importer/issues_importer.rb @@ -33,13 +33,17 @@ module Gitlab end def id_for_already_imported_cache(issue) - issue.number + issue[:number] end def collection_options { state: 'all', sort: 'created', direction: 'asc' } end + def increment_object_counter?(object) + object[:pull_request].nil? + end + private def additional_object_data diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb index 7293de56a9a..9a011f17a18 100644 --- a/lib/gitlab/github_import/importer/labels_importer.rb +++ b/lib/gitlab/github_import/importer/labels_importer.rb @@ -22,7 +22,7 @@ module Gitlab end def already_imported?(label) - existing_labels.include?(label.name) + existing_labels.include?(label[:name]) end def build_labels_cache @@ -33,8 +33,8 @@ module Gitlab time = Time.zone.now { - title: label.name, - color: '#' + label.color, + title: label[:name], + color: '#' + label[:color], project_id: project.id, type: 'ProjectLabel', created_at: time, diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb index d11b151bbe2..1a3a54d0053 100644 --- a/lib/gitlab/github_import/importer/milestones_importer.rb +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -22,7 +22,7 @@ module Gitlab end def already_imported?(milestone) - existing_milestones.include?(milestone.number) + existing_milestones.include?(milestone[:number]) end def build_milestones_cache @@ -31,19 +31,19 @@ module Gitlab def build(milestone) { - iid: milestone.number, - title: milestone.title, - description: milestone.description, + iid: milestone[:number], + title: milestone[:title], + description: milestone[:description], project_id: project.id, state: state_for(milestone), - due_date: milestone.due_on&.to_date, - created_at: milestone.created_at, - updated_at: milestone.updated_at + due_date: milestone[:due_on]&.to_date, + created_at: milestone[:created_at], + updated_at: milestone[:updated_at] } end def state_for(milestone) - milestone.state == 'open' ? :active : :closed + milestone[:state] == 'open' ? :active : :closed end def each_milestone diff --git a/lib/gitlab/github_import/importer/note_attachments_importer.rb b/lib/gitlab/github_import/importer/note_attachments_importer.rb new file mode 100644 index 00000000000..9901c9e76f5 --- /dev/null +++ b/lib/gitlab/github_import/importer/note_attachments_importer.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class NoteAttachmentsImporter + attr_reader :note_text, :project + + # note_text - An instance of `NoteText`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(note_text, project, _client = nil) + @note_text = note_text + @project = project + end + + def execute + attachments = MarkdownText.fetch_attachments(note_text.text) + return if attachments.blank? + + new_text = attachments.reduce(note_text.text) do |text, attachment| + new_url = download_attachment(attachment) + text.gsub(attachment.url, new_url) + end + + update_note_record(new_text) + end + + private + + # in: an instance of Gitlab::GithubImport::Markdown::Attachment + # out: gitlab attachment markdown url + def download_attachment(attachment) + downloader = ::Gitlab::GithubImport::AttachmentsDownloader.new(attachment.url) + file = downloader.perform + uploader = UploadService.new(project, file, FileUploader).execute + uploader.to_h[:url] + ensure + downloader&.delete + end + + def update_note_record(text) + case note_text.record_type + when ::Release.name + ::Release.find(note_text.record_db_id).update_column(:description, text) + when ::Issue.name + ::Issue.find(note_text.record_db_id).update_column(:description, text) + when ::MergeRequest.name + ::MergeRequest.find(note_text.record_db_id).update_column(:description, text) + when ::Note.name + ::Note.find(note_text.record_db_id).update_column(:note, text) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 1410006af26..69b7b2c2a38 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -36,6 +36,9 @@ module Gitlab # We're using bulk_insert here so we can bypass any validations and # callbacks. Running these would result in a lot of unnecessary SQL # queries being executed when importing large projects. + # Note: if you're going to replace `legacy_bulk_insert` with something that trigger callback + # to generate HTML version - you also need to regenerate it in + # Gitlab::GithubImport::Importer::NoteAttachmentsImporter. ApplicationRecord.legacy_bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert rescue ActiveRecord::InvalidForeignKey # It's possible the project and the issue have been deleted since diff --git a/lib/gitlab/github_import/importer/notes_importer.rb b/lib/gitlab/github_import/importer/notes_importer.rb index ca1d7d60515..4c2b87a8c5e 100644 --- a/lib/gitlab/github_import/importer/notes_importer.rb +++ b/lib/gitlab/github_import/importer/notes_importer.rb @@ -27,7 +27,7 @@ module Gitlab end def id_for_already_imported_cache(note) - note.id + note[:id] end end end diff --git a/lib/gitlab/github_import/importer/protected_branch_importer.rb b/lib/gitlab/github_import/importer/protected_branch_importer.rb index 16215fdce8e..21075e21e1d 100644 --- a/lib/gitlab/github_import/importer/protected_branch_importer.rb +++ b/lib/gitlab/github_import/importer/protected_branch_importer.rb @@ -6,6 +6,10 @@ module Gitlab class ProtectedBranchImporter attr_reader :protected_branch, :project, :client + # By default on GitHub, both developers and maintainers can merge + # a PR into the protected branch + GITHUB_DEFAULT_MERGE_ACCESS_LEVEL = Gitlab::Access::DEVELOPER + # protected_branch - An instance of # `Gitlab::GithubImport::Representation::ProtectedBranch`. # project - An instance of `Project` @@ -22,6 +26,8 @@ module Gitlab ProtectedBranches::CreateService .new(project, project.creator, params) .execute(skip_authorization: true) + + update_project_settings if default_branch? end private @@ -29,8 +35,8 @@ module Gitlab def params { name: protected_branch.id, - push_access_levels_attributes: [{ access_level: Gitlab::Access::MAINTAINER }], - merge_access_levels_attributes: [{ access_level: Gitlab::Access::MAINTAINER }], + push_access_levels_attributes: [{ access_level: push_access_level }], + merge_access_levels_attributes: [{ access_level: merge_access_level }], allow_force_push: allow_force_push? } end @@ -42,6 +48,95 @@ module Gitlab protected_branch.allow_force_pushes end end + + def default_branch? + protected_branch.id == project.default_branch + end + + def update_project_settings + update_setting_for_only_allow_merge_if_all_discussions_are_resolved + update_project_push_rule + end + + def update_setting_for_only_allow_merge_if_all_discussions_are_resolved + return unless protected_branch.required_conversation_resolution + + project.update(only_allow_merge_if_all_discussions_are_resolved: true) + end + + def update_project_push_rule + return unless project.licensed_feature_available?(:push_rules) + return unless protected_branch.required_signatures + + push_rule = project.push_rule || project.build_push_rule + push_rule.update!(reject_unsigned_commits: true) + project.project_setting.update!(push_rule_id: push_rule.id) + end + + def push_access_level + if protected_branch.required_pull_request_reviews + Gitlab::Access::NO_ACCESS + else + gitlab_access_level_for(:push) + end + end + + # Gets the strictest merge_access_level between GitHub and GitLab + def merge_access_level + gitlab_access = gitlab_access_level_for(:merge) + + return gitlab_access if gitlab_access == Gitlab::Access::NO_ACCESS + + [gitlab_access, GITHUB_DEFAULT_MERGE_ACCESS_LEVEL].max + end + + # action - :push/:merge + def gitlab_access_level_for(action) + if default_branch? + action == :push ? default_branch_push_access_level : default_branch_merge_access_level + elsif protected_on_gitlab? + non_default_branch_access_level_for(action) + else + gitlab_default_access_level_for(action) + end + end + + def default_branch_push_access_level + if default_branch_protection.developer_can_push? + Gitlab::Access::DEVELOPER + else + gitlab_default_access_level_for(:push) + end + end + + def default_branch_merge_access_level + if default_branch_protection.developer_can_merge? + Gitlab::Access::DEVELOPER + else + gitlab_default_access_level_for(:merge) + end + end + + def default_branch_protection + Gitlab::Access::BranchProtection.new(project.namespace.default_branch_protection) + end + + def protected_on_gitlab? + ProtectedBranch.protected?(project, protected_branch.id) + end + + def non_default_branch_access_level_for(action) + access_level = ProtectedBranch.access_levels_for_ref(protected_branch.id, action: action) + .find(&:role?)&.access_level + + access_level || gitlab_default_access_level_for(action) + end + + def gitlab_default_access_level_for(action) + return ProtectedBranch::PushAccessLevel::GITLAB_DEFAULT_ACCESS_LEVEL if action == :push + + ProtectedBranch::MergeAccessLevel::GITLAB_DEFAULT_ACCESS_LEVEL + end end end end diff --git a/lib/gitlab/github_import/importer/protected_branches_importer.rb b/lib/gitlab/github_import/importer/protected_branches_importer.rb index b5be823d5ab..4372477f55d 100644 --- a/lib/gitlab/github_import/importer/protected_branches_importer.rb +++ b/lib/gitlab/github_import/importer/protected_branches_importer.rb @@ -11,9 +11,9 @@ module Gitlab def each_object_to_import repo = project.import_source - protected_branches = client.branches(repo).select { |branch| branch.protection&.enabled } + protected_branches = client.branches(repo).select { |branch| branch.dig(:protection, :enabled) } protected_branches.each do |protected_branch| - object = client.branch_protection(repo, protected_branch.name) + object = client.branch_protection(repo, protected_branch[:name]) next if object.nil? || already_imported?(object) yield object @@ -44,7 +44,7 @@ module Gitlab end def id_for_already_imported_cache(protected_branch) - protected_branch.name + protected_branch[:name] end end end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index 5d291d9d723..16541c90002 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -19,7 +19,7 @@ module Gitlab end def id_for_already_imported_cache(pr) - pr.number + pr[:number] end def object_type @@ -55,11 +55,11 @@ module Gitlab def update_repository?(pr) last_update = project.last_repository_updated_at || project.created_at - return false if pr.updated_at < last_update + return false if pr[:updated_at] < last_update # PRs may be updated without there actually being new commits, thus we # check to make sure we only re-fetch if truly necessary. - !(commit_exists?(pr.head.sha) && commit_exists?(pr.base.sha)) + !(commit_exists?(pr.dig(:head, :sha)) && commit_exists?(pr.dig(:base, :sha))) end def commit_exists?(sha) diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb index 5e55d09fe3d..543c29a21a0 100644 --- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb @@ -34,7 +34,7 @@ module Gitlab end def id_for_already_imported_cache(review) - review.id + review[:id] end # The worker can be interrupted, by rate limit for instance, @@ -48,11 +48,13 @@ module Gitlab def each_object_to_import(&block) each_review_page do |page, merge_request| page.objects.each do |review| + review = review.to_h + next if already_imported?(review) Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - review.merge_request_id = merge_request.id + review[:merge_request_id] = merge_request.id yield(review) mark_as_imported(review) diff --git a/lib/gitlab/github_import/importer/release_attachments_importer.rb b/lib/gitlab/github_import/importer/release_attachments_importer.rb deleted file mode 100644 index 6419851623c..00000000000 --- a/lib/gitlab/github_import/importer/release_attachments_importer.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GithubImport - module Importer - class ReleaseAttachmentsImporter - attr_reader :release_db_id, :release_description, :project - - # release - An instance of `ReleaseAttachments`. - # project - An instance of `Project`. - # client - An instance of `Gitlab::GithubImport::Client`. - def initialize(release_attachments, project, _client = nil) - @release_db_id = release_attachments.release_db_id - @release_description = release_attachments.description - @project = project - end - - def execute - attachment_urls = MarkdownText.fetch_attachment_urls(release_description) - new_description = attachment_urls.reduce(release_description) do |description, url| - new_url = download_attachment(url) - description.gsub(url, new_url) - end - - Release.find(release_db_id).update_column(:description, new_description) - end - - private - - # in: github attachment markdown url - # out: gitlab attachment markdown url - def download_attachment(markdown_url) - url = extract_url_from_markdown(markdown_url) - name_prefix = extract_name_from_markdown(markdown_url) - - downloader = ::Gitlab::GithubImport::AttachmentsDownloader.new(url) - file = downloader.perform - uploader = UploadService.new(project, file, FileUploader).execute - "#{name_prefix}(#{uploader.to_h[:url]})" - ensure - downloader&.delete - end - - # in: "" - # out: https://user-images.githubusercontent.com/.. - def extract_url_from_markdown(text) - text.match(%r{https://.*\)$}).to_a[0].chop - end - - # in: "" - # out: ![image-icon] - def extract_name_from_markdown(text) - text.match(%r{^!?\[.*\]}).to_a[0] - end - end - end - end -end diff --git a/lib/gitlab/github_import/importer/releases_attachments_importer.rb b/lib/gitlab/github_import/importer/releases_attachments_importer.rb deleted file mode 100644 index 7221c802d83..00000000000 --- a/lib/gitlab/github_import/importer/releases_attachments_importer.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GithubImport - module Importer - class ReleasesAttachmentsImporter - include ParallelScheduling - - BATCH_SIZE = 100 - - # The method that will be called for traversing through all the objects to - # import, yielding them to the supplied block. - def each_object_to_import - project.releases.select(:id, :description).each_batch(of: BATCH_SIZE, column: :id) do |batch| - batch.each do |release| - next if already_imported?(release) - - Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - - yield release - - # We mark the object as imported immediately so we don't end up - # scheduling it multiple times. - mark_as_imported(release) - end - end - end - - def representation_class - Representation::ReleaseAttachments - end - - def importer_class - ReleaseAttachmentsImporter - end - - def sidekiq_worker_class - ImportReleaseAttachmentsWorker - end - - def collection_method - :release_attachments - end - - def object_type - :release_attachment - end - - def id_for_already_imported_cache(release) - release.id - end - - def object_representation(object) - representation_class.from_db_record(object) - end - end - end - end -end diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index 51d364772d2..fe6da30bbf8 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -12,6 +12,9 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # Note: if you're going to replace `legacy_bulk_insert` with something that triggers callback + # to generate HTML version - you also need to regenerate it in + # Gitlab::GithubImport::Importer::NoteAttachmentsImporter. def execute bulk_insert(Release, build_releases) end @@ -21,21 +24,21 @@ module Gitlab end def already_imported?(release) - existing_tags.include?(release.tag_name) || release.tag_name.nil? + existing_tags.include?(release[:tag_name]) || release[:tag_name].nil? end def build(release) - existing_tags.add(release.tag_name) + existing_tags.add(release[:tag_name]) { - name: release.name, - tag: release.tag_name, + name: release[:name], + tag: release[:tag_name], author_id: fetch_author_id(release), description: description_for(release), - created_at: release.created_at, - updated_at: release.created_at, + created_at: release[:created_at], + updated_at: release[:created_at], # Draft releases will have a null published_at - released_at: release.published_at || Time.current, + released_at: release[:published_at] || Time.current, project_id: project.id } end @@ -45,7 +48,7 @@ module Gitlab end def description_for(release) - release.body.presence || "Release for tag #{release.tag_name}" + release[:body].presence || "Release for tag #{release[:tag_name]}" end def object_type diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb index 8a9ddfc6ec0..4090555c85e 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -22,13 +22,15 @@ module Gitlab # To make it possible to identify issue in separated worker we need to patch # Sawyer instances here with issue number def each_associated(parent_record, associated) + associated = associated.to_h + compose_associated_id!(parent_record, associated) return if already_imported?(associated) Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) pull_request = parent_record.is_a? MergeRequest - associated.issue = { 'number' => parent_record.iid, 'pull_request' => pull_request } + associated[:issue] = { number: parent_record.iid, pull_request: pull_request } yield(associated) mark_as_imported(associated) @@ -78,7 +80,7 @@ module Gitlab end def id_for_already_imported_cache(event) - event.id + event[:id] end def collection_options @@ -87,9 +89,9 @@ module Gitlab # Cross-referenced events on Github doesn't have id. def compose_associated_id!(issuable, event) - return if event.event != 'cross-referenced' + return if event[:event] != 'cross-referenced' - event.id = "cross-reference##{issuable.iid}-in-#{event.source.issue.id}" + event[:id] = "cross-reference##{issuable.iid}-in-#{event.dig(:source, :issue, :id)}" end end end diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb index e7a1b7b3368..b960df581e4 100644 --- a/lib/gitlab/github_import/issuable_finder.rb +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -80,12 +80,16 @@ module Gitlab end def timeout - if project.group.present? && ::Feature.enabled?(:github_importer_single_endpoint_notes_import, project.group, type: :ops) + if import_settings.enabled?(:single_endpoint_notes_import) Gitlab::Cache::Import::Caching::LONGER_TIMEOUT else Gitlab::Cache::Import::Caching::TIMEOUT end end + + def import_settings + ::Gitlab::GithubImport::Settings.new(project) + end end end end diff --git a/lib/gitlab/github_import/markdown/attachment.rb b/lib/gitlab/github_import/markdown/attachment.rb new file mode 100644 index 00000000000..a5cf5ffa60e --- /dev/null +++ b/lib/gitlab/github_import/markdown/attachment.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Markdown + class Attachment + MEDIA_TYPES = %w[gif jpeg jpg mov mp4 png svg webm].freeze + DOC_TYPES = %w[ + csv docx fodg fodp fods fodt gz log md odf odg odp ods + odt pdf pptx tgz txt xls xlsx zip + ].freeze + + class << self + # markdown_node - CommonMarker::Node + def from_markdown(markdown_node) + case markdown_node.type + when :html, :inline_html + from_inline_html(markdown_node) + when :image + from_markdown_image(markdown_node) + when :link + from_markdown_link(markdown_node) + end + end + + private + + def from_markdown_image(markdown_node) + url = markdown_node.url + + return unless github_url?(url, media: true) + return unless whitelisted_type?(url, media: true) + + new(markdown_node.to_plaintext.strip, url) + end + + def from_markdown_link(markdown_node) + url = markdown_node.url + + return unless github_url?(url, docs: true) + return unless whitelisted_type?(url, docs: true) + + new(markdown_node.to_plaintext.strip, url) + end + + def from_inline_html(markdown_node) + img = Nokogiri::HTML.parse(markdown_node.string_content).xpath('//img')[0] + + return unless img + return unless github_url?(img[:src], media: true) + return unless whitelisted_type?(img[:src], media: true) + + new(img[:alt], img[:src]) + end + + def github_url?(url, docs: false, media: false) + if media + url.start_with?(::Gitlab::GithubImport::MarkdownText::GITHUB_MEDIA_CDN) + elsif docs + url.start_with?(::Gitlab::GithubImport::MarkdownText.github_url) + end + end + + def whitelisted_type?(url, docs: false, media: false) + if media + MEDIA_TYPES.any? { |type| url.end_with?(type) } + elsif docs + DOC_TYPES.any? { |type| url.end_with?(type) } + end + end + end + + attr_reader :name, :url + + def initialize(name, url) + @name = name + @url = url + end + + def inspect + "<#{self.class.name}: { name: #{name}, url: #{url} }>" + end + end + end + end +end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index bf2856bc77f..2424b3e8c1f 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -8,23 +8,12 @@ module Gitlab class MarkdownText include Gitlab::EncodingHelper - ISSUE_REF_MATCHER = '%{github_url}/%{import_source}/issues' - PULL_REF_MATCHER = '%{github_url}/%{import_source}/pull' - - MEDIA_TYPES = %w[gif jpeg jpg mov mp4 png svg webm].freeze - DOC_TYPES = %w[ - csv docx fodg fodp fods fodt gz log md odf odg odp ods - odt pdf pptx tgz txt xls xlsx zip - ].freeze - ALL_TYPES = (MEDIA_TYPES + DOC_TYPES).freeze - # On github.com we have base url for docs and CDN url for media. # On github EE as far as we can know there is no CDN urls and media is placed on base url. - # To no escape the escaping symbol we use single quotes instead of double with interpolation. - # rubocop:disable Style/StringConcatenation - CDN_URL_MATCHER = '(!\[.+\]\(%{github_media_cdn}/\d+/(\w|-)+\.(' + MEDIA_TYPES.join('|') + ')\))' - BASE_URL_MATCHER = '(\[.+\]\(%{github_url}/.+/.+/files/\d+/.+\.(' + ALL_TYPES.join('|') + ')\))' - # rubocop:enable Style/StringConcatenation + GITHUB_MEDIA_CDN = 'https://user-images.githubusercontent.com' + + ISSUE_REF_MATCHER = '%{github_url}/%{import_source}/issues' + PULL_REF_MATCHER = '%{github_url}/%{import_source}/pull' class << self def format(*args) @@ -42,20 +31,6 @@ module Gitlab .gsub(pull_ref_matcher, url_helpers.project_merge_requests_url(project)) end - def fetch_attachment_urls(text) - cdn_url_matcher = CDN_URL_MATCHER % { github_media_cdn: Regexp.escape(github_media_cdn) } - doc_url_matcher = BASE_URL_MATCHER % { github_url: Regexp.escape(github_url) } - - text.scan(Regexp.new(cdn_url_matcher)).map(&:first) + - text.scan(Regexp.new(doc_url_matcher)).map(&:first) - end - - private - - def github_media_cdn - 'https://user-images.githubusercontent.com' - end - # Returns github domain without slash in the end def github_url oauth_config = Gitlab::Auth::OAuth::Provider.config_for('github') || {} @@ -63,6 +38,23 @@ module Gitlab url = url.chop if url.end_with?('/') url end + + def fetch_attachments(text) + attachments = [] + doc = CommonMarker.render_doc(text) + + doc.walk do |node| + attachment = extract_attachment(node) + attachments << attachment if attachment + end + attachments + end + + private + + def extract_attachment(node) + ::Gitlab::GithubImport::Markdown::Attachment.from_markdown(node) + end end # text - The Markdown text as a String. diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index bf5046de36c..03aa02fb659 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -125,9 +125,13 @@ module Gitlab next unless page_counter.set(page.number) page.objects.each do |object| + object = object.to_h + next if already_imported?(object) - Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + if increment_object_counter?(object) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + end yield object @@ -138,6 +142,10 @@ module Gitlab end end + def increment_object_counter?(_object) + true + end + # Returns true if the given object has already been imported, false # otherwise. # diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index 64aa6ea5cb4..f3be90834c7 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -19,33 +19,33 @@ module Gitlab # Builds a diff note from a GitHub API response. # - # note - An instance of `Sawyer::Resource` containing the note details. + # note - An instance of `Hash` containing the note details. def self.from_api_response(note, additional_data = {}) - matches = note.html_url.match(NOTEABLE_ID_REGEX) + matches = note[:html_url].match(NOTEABLE_ID_REGEX) unless matches raise( ArgumentError, - "The note URL #{note.html_url.inspect} is not supported" + "The note URL #{note[:html_url].inspect} is not supported" ) end - user = Representation::User.from_api_response(note.user) if note.user + user = Representation::User.from_api_response(note[:user]) if note[:user] hash = { noteable_id: matches[:iid].to_i, - file_path: note.path, - commit_id: note.commit_id, - original_commit_id: note.original_commit_id, - diff_hunk: note.diff_hunk, + file_path: note[:path], + commit_id: note[:commit_id], + original_commit_id: note[:original_commit_id], + diff_hunk: note[:diff_hunk], author: user, - note: note.body, - created_at: note.created_at, - updated_at: note.updated_at, - note_id: note.id, - end_line: note.line, - start_line: note.start_line, - side: note.side, - in_reply_to_id: note.in_reply_to_id + note: note[:body], + created_at: note[:created_at], + updated_at: note[:updated_at], + note_id: note[:id], + end_line: note[:line], + start_line: note[:start_line], + side: note[:side], + in_reply_to_id: note[:in_reply_to_id] } new(hash) diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb index 9d457ec1c2f..e878aeaf3b9 100644 --- a/lib/gitlab/github_import/representation/issue.rb +++ b/lib/gitlab/github_import/representation/issue.rb @@ -15,28 +15,28 @@ module Gitlab # Builds an issue from a GitHub API response. # - # issue - An instance of `Sawyer::Resource` containing the issue + # issue - An instance of `Hash` containing the issue # details. def self.from_api_response(issue, additional_data = {}) user = - if issue.user - Representation::User.from_api_response(issue.user) + if issue[:user] + Representation::User.from_api_response(issue[:user]) end hash = { - iid: issue.number, - title: issue.title, - description: issue.body, - milestone_number: issue.milestone&.number, - state: issue.state == 'open' ? :opened : :closed, - assignees: issue.assignees.map do |u| + iid: issue[:number], + title: issue[:title], + description: issue[:body], + milestone_number: issue.dig(:milestone, :number), + state: issue[:state] == 'open' ? :opened : :closed, + assignees: issue[:assignees].map do |u| Representation::User.from_api_response(u) end, - label_names: issue.labels.map(&:name), + label_names: issue[:labels].map { _1[:name] }, author: user, - created_at: issue.created_at, - updated_at: issue.updated_at, - pull_request: issue.pull_request ? true : false, + created_at: issue[:created_at], + updated_at: issue[:updated_at], + pull_request: issue[:pull_request] ? true : false, work_item_type_id: additional_data[:work_item_type_id] } diff --git a/lib/gitlab/github_import/representation/issue_event.rb b/lib/gitlab/github_import/representation/issue_event.rb index 89271a7dcd6..39a23c016ce 100644 --- a/lib/gitlab/github_import/representation/issue_event.rb +++ b/lib/gitlab/github_import/representation/issue_event.rb @@ -34,23 +34,23 @@ module Gitlab class << self # Builds an event from a GitHub API response. # - # event - An instance of `Sawyer::Resource` containing the event details. + # event - An instance of `Hash` containing the event details. def from_api_response(event, additional_data = {}) new( - id: event.id, - actor: user_representation(event.actor), - event: event.event, - commit_id: event.commit_id, - label_title: event.label && event.label[:name], - old_title: event.rename && event.rename[:from], - new_title: event.rename && event.rename[:to], - milestone_title: event.milestone && event.milestone[:title], - issue: event.issue&.to_h&.symbolize_keys, - source: event.source, - assignee: user_representation(event.assignee), - requested_reviewer: user_representation(event.requested_reviewer), - review_requester: user_representation(event.review_requester), - created_at: event.created_at + id: event[:id], + actor: user_representation(event[:actor]), + event: event[:event], + commit_id: event[:commit_id], + label_title: event.dig(:label, :name), + old_title: event.dig(:rename, :from), + new_title: event.dig(:rename, :to), + milestone_title: event.dig(:milestone, :title), + issue: event[:issue], + source: event[:source], + assignee: user_representation(event[:assignee]), + requested_reviewer: user_representation(event[:requested_reviewer]), + review_requester: user_representation(event[:review_requester]), + created_at: event[:created_at] ) end diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index ae56c370b19..14379e8a4e9 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -16,14 +16,14 @@ module Gitlab # Builds a note from a GitHub API response. # - # note - An instance of `Sawyer::Resource` containing the note details. + # note - An instance of `Hash` containing the note details. def self.from_api_response(note, additional_data = {}) - matches = note.html_url.match(NOTEABLE_TYPE_REGEX) + matches = note[:html_url].match(NOTEABLE_TYPE_REGEX) if !matches || !matches[:type] raise( ArgumentError, - "The note URL #{note.html_url.inspect} is not supported" + "The note URL #{note[:html_url].inspect} is not supported" ) end @@ -34,15 +34,15 @@ module Gitlab 'Issue' end - user = Representation::User.from_api_response(note.user) if note.user + user = Representation::User.from_api_response(note[:user]) if note[:user] hash = { noteable_type: noteable_type, noteable_id: matches[:iid].to_i, author: user, - note: note.body, - created_at: note.created_at, - updated_at: note.updated_at, - note_id: note.id + note: note[:body], + created_at: note[:created_at], + updated_at: note[:updated_at], + note_id: note[:id] } new(hash) diff --git a/lib/gitlab/github_import/representation/note_text.rb b/lib/gitlab/github_import/representation/note_text.rb new file mode 100644 index 00000000000..505d7d805d3 --- /dev/null +++ b/lib/gitlab/github_import/representation/note_text.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +# This class only partly represents MODELS_ALLOWLIST records from DB and +# is used to connect ReleasesAttachmentsImporter, NotesAttachmentsImporter etc. +# with NoteAttachmentsImporter without modifying ObjectImporter a lot. +# Attachments are inside release's `description`. +module Gitlab + module GithubImport + module Representation + class NoteText + include ToHash + include ExposeAttribute + + MODELS_ALLOWLIST = [::Release, ::Note, ::Issue, ::MergeRequest].freeze + ModelNotSupported = Class.new(StandardError) + + attr_reader :attributes + + expose_attribute :record_db_id, :record_type, :text + + class << self + # Builds a note text representation from DB record of Note or Release. + # + # record - An instance of `Note`, `Release`, `Issue`, `MergeRequest` model + def from_db_record(record) + check_record_class!(record) + + record_type = record.class.name + # only column for note is different along MODELS_ALLOWLIST + text = record.is_a?(::Note) ? record.note : record.description + new( + record_db_id: record.id, + record_type: record_type, + text: text + ) + end + + def from_json_hash(raw_hash) + new Representation.symbolize_hash(raw_hash) + end + + private + + def check_record_class!(record) + raise ModelNotSupported, record.class.name if MODELS_ALLOWLIST.exclude?(record.class) + end + end + + # attributes - A Hash containing the event details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { db_id: record_db_id } + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/protected_branch.rb b/lib/gitlab/github_import/representation/protected_branch.rb index b80b7cf1076..07a607ae70d 100644 --- a/lib/gitlab/github_import/representation/protected_branch.rb +++ b/lib/gitlab/github_import/representation/protected_branch.rb @@ -9,18 +9,22 @@ module Gitlab attr_reader :attributes - expose_attribute :id, :allow_force_pushes + expose_attribute :id, :allow_force_pushes, :required_conversation_resolution, :required_signatures, + :required_pull_request_reviews # Builds a Branch Protection info from a GitHub API response. # Resource structure details: # https://docs.github.com/en/rest/branches/branch-protection#get-branch-protection - # branch_protection - An instance of `Sawyer::Resource` containing the protection details. + # branch_protection - An instance of `Hash` containing the protection details. def self.from_api_response(branch_protection, _additional_object_data = {}) - branch_name = branch_protection.url.match(%r{/branches/(\S{1,255})/protection$})[1] + branch_name = branch_protection[:url].match(%r{/branches/(\S{1,255})/protection$})[1] hash = { id: branch_name, - allow_force_pushes: branch_protection.allow_force_pushes.enabled + allow_force_pushes: branch_protection.dig(:allow_force_pushes, :enabled), + required_conversation_resolution: branch_protection.dig(:required_conversation_resolution, :enabled), + required_signatures: branch_protection.dig(:required_signatures, :enabled), + required_pull_request_reviews: branch_protection[:required_pull_request_reviews].present? } new(hash) diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb index 2adac2af502..4b8ae1f8eab 100644 --- a/lib/gitlab/github_import/representation/pull_request.rb +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -17,30 +17,30 @@ module Gitlab # Builds a PR from a GitHub API response. # - # issue - An instance of `Sawyer::Resource` containing the PR details. + # issue - An instance of `Hash` containing the PR details. def self.from_api_response(pr, additional_data = {}) - assignee = Representation::User.from_api_response(pr.assignee) if pr.assignee - user = Representation::User.from_api_response(pr.user) if pr.user - merged_by = Representation::User.from_api_response(pr.merged_by) if pr.merged_by + assignee = Representation::User.from_api_response(pr[:assignee]) if pr[:assignee] + user = Representation::User.from_api_response(pr[:user]) if pr[:user] + merged_by = Representation::User.from_api_response(pr[:merged_by]) if pr[:merged_by] hash = { - iid: pr.number, - title: pr.title, - description: pr.body, - source_branch: pr.head.ref, - target_branch: pr.base.ref, - source_branch_sha: pr.head.sha, - target_branch_sha: pr.base.sha, - source_repository_id: pr.head&.repo&.id, - target_repository_id: pr.base&.repo&.id, - source_repository_owner: pr.head&.user&.login, - state: pr.state == 'open' ? :opened : :closed, - milestone_number: pr.milestone&.number, + iid: pr[:number], + title: pr[:title], + description: pr[:body], + source_branch: pr.dig(:head, :ref), + target_branch: pr.dig(:base, :ref), + source_branch_sha: pr.dig(:head, :sha), + target_branch_sha: pr.dig(:base, :sha), + source_repository_id: pr.dig(:head, :repo, :id), + target_repository_id: pr.dig(:base, :repo, :id), + source_repository_owner: pr.dig(:head, :user, :login), + state: pr[:state] == 'open' ? :opened : :closed, + milestone_number: pr.dig(:milestone, :number), author: user, assignee: assignee, - created_at: pr.created_at, - updated_at: pr.updated_at, - merged_at: pr.merged_at, + created_at: pr[:created_at], + updated_at: pr[:updated_at], + merged_at: pr[:merged_at], merged_by: merged_by } diff --git a/lib/gitlab/github_import/representation/pull_request_review.rb b/lib/gitlab/github_import/representation/pull_request_review.rb index 8a7ecf0c588..8fb57ae89a4 100644 --- a/lib/gitlab/github_import/representation/pull_request_review.rb +++ b/lib/gitlab/github_import/representation/pull_request_review.rb @@ -11,16 +11,19 @@ module Gitlab expose_attribute :author, :note, :review_type, :submitted_at, :merge_request_id, :review_id + # Builds a PullRequestReview from a GitHub API response. + # + # review - An instance of `Hash` containing the note details. def self.from_api_response(review, additional_data = {}) - user = Representation::User.from_api_response(review.user) if review.user + user = Representation::User.from_api_response(review[:user]) if review[:user] new( - merge_request_id: review.merge_request_id, + merge_request_id: review[:merge_request_id], author: user, - note: review.body, - review_type: review.state, - submitted_at: review.submitted_at, - review_id: review.id + note: review[:body], + review_type: review[:state], + submitted_at: review[:submitted_at], + review_id: review[:id] ) end diff --git a/lib/gitlab/github_import/representation/release_attachments.rb b/lib/gitlab/github_import/representation/release_attachments.rb deleted file mode 100644 index fd272be2405..00000000000 --- a/lib/gitlab/github_import/representation/release_attachments.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -# This class only partly represents Release record from DB and -# is used to connect ReleasesAttachmentsImporter with ReleaseAttachmentsImporter -# without modifying ObjectImporter a lot. -# Attachments are inside release's `description`. -module Gitlab - module GithubImport - module Representation - class ReleaseAttachments - include ToHash - include ExposeAttribute - - attr_reader :attributes - - expose_attribute :release_db_id, :description - - # Builds a event from a GitHub API response. - # - # release - An instance of `Release` model. - def self.from_db_record(release) - new( - release_db_id: release.id, - description: release.description - ) - end - - def self.from_json_hash(raw_hash) - new Representation.symbolize_hash(raw_hash) - end - - # attributes - A Hash containing the event details. The keys of this - # Hash (and any nested hashes) must be symbols. - def initialize(attributes) - @attributes = attributes - end - - def github_identifiers - { db_id: release_db_id } - end - end - end - end -end diff --git a/lib/gitlab/github_import/representation/user.rb b/lib/gitlab/github_import/representation/user.rb index 4ef916cc41c..02cbe037384 100644 --- a/lib/gitlab/github_import/representation/user.rb +++ b/lib/gitlab/github_import/representation/user.rb @@ -13,11 +13,11 @@ module Gitlab # Builds a user from a GitHub API response. # - # user - An instance of `Sawyer::Resource` containing the user details. + # user - An instance of `Hash` containing the user details. def self.from_api_response(user, additional_data = {}) new( - id: user.id, - login: user.login + id: user[:id], + login: user[:login] ) end diff --git a/lib/gitlab/github_import/settings.rb b/lib/gitlab/github_import/settings.rb new file mode 100644 index 00000000000..77288b9fb98 --- /dev/null +++ b/lib/gitlab/github_import/settings.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class Settings + OPTIONAL_STAGES = { + single_endpoint_issue_events_import: { + label: 'Import issue and pull request events', + details: <<-TEXT.split("\n").map(&:strip).join(' ') + For example, opened or closed, renamed, and labeled or unlabeled. + Time required to import these events depends on how many issues or pull requests your project has. + TEXT + }, + single_endpoint_notes_import: { + label: 'Use alternative comments import method', + details: <<-TEXT.split("\n").map(&:strip).join(' ') + The default method can skip some comments in large projects because of limitations of the GitHub API. + TEXT + }, + attachments_import: { + label: 'Import Markdown attachments', + details: <<-TEXT.split("\n").map(&:strip).join(' ') + Import Markdown attachments from repository comments, release posts, issue descriptions, + and pull request descriptions. These can include images, text, or binary attachments. + If not imported, links in Markdown to attachments break after you remove the attachments from GitHub. + TEXT + } + }.freeze + + def self.stages_array + OPTIONAL_STAGES.map do |stage_name, data| + { + name: stage_name.to_s, + label: s_(format("GitHubImport|%{text}", text: data[:label])), + details: s_(format("GitHubImport|%{text}", text: data[:details])) + } + end + end + + def initialize(project) + @project = project + end + + def write(user_settings) + user_settings = user_settings.to_h.with_indifferent_access + + optional_stages = fetch_stages_from_params(user_settings) + import_data = project.create_or_update_import_data(data: { optional_stages: optional_stages }) + import_data.save! + end + + def enabled?(stage_name) + project.import_data&.data&.dig('optional_stages', stage_name.to_s) || false + end + + def disabled?(stage_name) + !enabled?(stage_name) + end + + private + + attr_reader :project + + def fetch_stages_from_params(user_settings) + OPTIONAL_STAGES.keys.to_h do |stage_name| + enabled = Gitlab::Utils.to_boolean(user_settings[stage_name], default: false) + [stage_name, enabled] + end + end + end + end +end diff --git a/lib/gitlab/github_import/single_endpoint_notes_importing.rb b/lib/gitlab/github_import/single_endpoint_notes_importing.rb index aea4059dfbc..3584288da57 100644 --- a/lib/gitlab/github_import/single_endpoint_notes_importing.rb +++ b/lib/gitlab/github_import/single_endpoint_notes_importing.rb @@ -4,10 +4,10 @@ # - SingleEndpointDiffNotesImporter # - SingleEndpointIssueNotesImporter # - SingleEndpointMergeRequestNotesImporter -# if `github_importer_single_endpoint_notes_import` feature flag is on. +# if enabled by Gitlab::GithubImport::Settings # # - SingleEndpointIssueEventsImporter -# if `github_importer_issue_events_import` feature flag is on. +# if enabled by Gitlab::GithubImport::Settings # # Fetches associated objects page by page to each item of parent collection. # Currently `associated` is note or event. @@ -32,7 +32,7 @@ module Gitlab end def id_for_already_imported_cache(associated) - associated.id + associated[:id] end def parent_collection @@ -54,6 +54,8 @@ module Gitlab # in Github API response object. For example: # lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb:26 def each_associated(_parent_record, associated) + associated = associated.to_h + return if already_imported?(associated) Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 1feb0d450f0..b8751def08f 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -39,18 +39,19 @@ module Gitlab # # If the object has no author ID we'll use the ID of the GitLab ghost # user. + # object - An instance of `Hash` or a `Github::Representer` def author_id_for(object, author_key: :author) user_info = case author_key when :actor - object&.actor + object[:actor] when :assignee - object&.assignee + object[:assignee] when :requested_reviewer - object&.requested_reviewer + object[:requested_reviewer] when :review_requester - object&.review_requester + object[:review_requester] else - object&.author + object ? object[:author] : nil end id = user_info ? user_id_for(user_info) : GithubImport.ghost_user_id @@ -64,14 +65,14 @@ module Gitlab # Returns the GitLab user ID of an issuable's assignee. def assignee_id_for(issuable) - user_id_for(issuable.assignee) if issuable.assignee + user_id_for(issuable[:assignee]) if issuable[:assignee] end # Returns the GitLab user ID for a GitHub user. # - # user - An instance of `Gitlab::GithubImport::Representation::User`. + # user - An instance of `Gitlab::GithubImport::Representation::User` or `Hash`. def user_id_for(user) - find(user.id, user.login) if user.present? + find(user[:id], user[:login]) if user.present? end # Returns the GitLab ID for the given GitHub ID or username. @@ -114,7 +115,7 @@ module Gitlab unless email user = client.user(username) - email = Gitlab::Cache::Import::Caching.write(cache_key, user.email, timeout: timeout(user.email)) if user + email = Gitlab::Cache::Import::Caching.write(cache_key, user[:email], timeout: timeout(user[:email])) if user end email diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 08a614edb4b..bdb7484f3d6 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -15,6 +15,7 @@ module Gitlab gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.markdown_surround_selection = current_user&.markdown_surround_selection + gon.markdown_automatic_lists = current_user&.markdown_automatic_lists if Gitlab.config.sentry.enabled gon.sentry_dsn = Gitlab.config.sentry.clientside_dsn @@ -32,6 +33,7 @@ module Gitlab gon.sprite_file_icons = IconsHelper.sprite_file_icons_path gon.emoji_sprites_css_path = ActionController::Base.helpers.stylesheet_path('emoji_sprites') gon.select2_css_path = ActionController::Base.helpers.stylesheet_path('lazy_bundles/select2.css') + gon.gridstack_css_path = ActionController::Base.helpers.stylesheet_path('lazy_bundles/gridstack.css') gon.test_env = Rails.env.test? gon.disable_animations = Gitlab.config.gitlab['disable_animations'] gon.suggested_label_colors = LabelsHelper.suggested_colors @@ -55,7 +57,7 @@ module Gitlab push_frontend_feature_flag(:security_auto_fix) push_frontend_feature_flag(:new_header_search) push_frontend_feature_flag(:source_editor_toolbar) - push_frontend_feature_flag(:gl_avatar_for_all_user_avatars) + push_frontend_feature_flag(:integration_slack_app_notifications) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index 987a5e7b74b..eca4d42fb9a 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -151,7 +151,7 @@ module Gitlab def limit_value # note: only first _or_ last can be specified, not both - @limit_value ||= [first, last, max_page_size, GitlabSchema.default_max_page_size].compact.min + @limit_value ||= [first, last, max_page_size || GitlabSchema.default_max_page_size].compact.min end def loaded?(items) diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 5b9216c0914..a2d06b7f5b3 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,28 +44,28 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 38, + 'da_DK' => 37, 'de' => 17, 'en' => 100, 'eo' => 0, - 'es' => 37, + 'es' => 36, 'fil_PH' => 0, - 'fr' => 11, + 'fr' => 72, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, 'ja' => 31, - 'ko' => 17, - 'nb_NO' => 26, + 'ko' => 20, + 'nb_NO' => 25, 'nl_NL' => 0, - 'pl_PL' => 4, - 'pt_BR' => 56, + 'pl_PL' => 3, + 'pt_BR' => 57, 'ro_RO' => 99, - 'ru' => 27, - 'si_LK' => 10, + 'ru' => 26, + 'si_LK' => 11, 'tr_TR' => 11, - 'uk' => 50, - 'zh_CN' => 97, + 'uk' => 49, + 'zh_CN' => 98, 'zh_HK' => 1, 'zh_TW' => 99 }.freeze diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index 6c5fba37d7b..fe0ab01e4fd 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -26,10 +26,10 @@ module Gitlab log_info(message: "Started uploading project", export_size: export_size) upload_duration = Benchmark.realtime do - if Feature.enabled?(:import_export_web_upload_stream) && !project.export_file.file_storage? - upload_project_as_remote_stream - else + if project.export_file.file_storage? handle_response_error(send_file) + else + upload_project_as_remote_stream end end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index bbec473d29d..b05d9cb2489 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -15,19 +15,19 @@ module Gitlab UNIQUE_RELATIONS = %i[].freeze USER_REFERENCES = %w[ - author_id - assignee_id - updated_by_id - merged_by_id - latest_closed_by_id - user_id - created_by_id - last_edited_by_id - merge_user_id - resolved_by_id - closed_by_id - owner_id - ].freeze + author_id + assignee_id + updated_by_id + merged_by_id + latest_closed_by_id + user_id + created_by_id + last_edited_by_id + merge_user_id + resolved_by_id + closed_by_id + owner_id + ].freeze TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook ErrorTracking::ProjectErrorTrackingSetting].freeze diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 99396d64779..cf62f181366 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -175,21 +175,22 @@ module Gitlab order_expression = arel_table[column].public_send(direction).public_send(nulls_position) # rubocop:disable GitlabSecurity/PublicSend reverse_order_expression = arel_table[column].public_send(reverse_direction).public_send(reverse_nulls_position) # rubocop:disable GitlabSecurity/PublicSend - ::Gitlab::Pagination::Keyset::Order.build([ - ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: column, - column_expression: arel_table[column], - order_expression: order_expression, - reversed_order_expression: reverse_order_expression, - order_direction: direction, - nullable: nulls_position, - distinct: false - ), - ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: klass.primary_key, - order_expression: arel_order_classes[direction].new(arel_table[klass.primary_key.to_sym]) - ) - ]) + ::Gitlab::Pagination::Keyset::Order.build( + [ + ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: column, + column_expression: arel_table[column], + order_expression: order_expression, + reversed_order_expression: reverse_order_expression, + order_direction: direction, + nullable: nulls_position, + distinct: false + ), + ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: klass.primary_key, + order_expression: arel_order_classes[direction].new(arel_table[klass.primary_key.to_sym]) + ) + ]) end def read_from_replica_if_available(&block) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 33e4823f192..fb44aaf094e 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -98,6 +98,7 @@ tree: - :statuses - :external_pull_request - :merge_request + - :pipeline_metadata - :auto_devops - :pipeline_schedules - :container_expiration_policy @@ -582,6 +583,9 @@ included_attributes: - :iid - :source_sha - :target_sha + pipeline_metadata: + - :project_id + - :title stages: - :name - :status @@ -971,6 +975,8 @@ excluded_attributes: - :external_pull_request_id - :ci_ref_id - :locked + pipeline_metadata: + - :pipeline_id stages: - :pipeline_id merge_access_levels: diff --git a/lib/gitlab/import_export/project/import_task.rb b/lib/gitlab/import_export/project/import_task.rb index 89f2b36ea58..4ea47a5624a 100644 --- a/lib/gitlab/import_export/project/import_task.rb +++ b/lib/gitlab/import_export/project/import_task.rb @@ -64,7 +64,7 @@ module Gitlab end def execute_sidekiq_job - Sidekiq::Worker.drain_all + Sidekiq::Worker.drain_all # rubocop:disable Cop/SidekiqApiUsage end def full_path diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index c4b0e24e34a..568315930d8 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -13,6 +13,7 @@ module Gitlab pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', runners: 'Ci::Runner', + pipeline_metadata: 'Ci::PipelineMetadata', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index ad19508fb99..bc0563729a7 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -86,8 +86,9 @@ module Gitlab mkdir_p(File.join(uploads_export_path, secret)) download_or_copy_upload(upload, upload_path) - rescue Errno::ENAMETOOLONG => e - # Do not fail entire project export if downloaded file has filename that exceeds 255 characters. + rescue StandardError => e + # Do not fail entire project export if something goes wrong during file download + # (e.g. downloaded file has filename that exceeds 255 characters). # Ignore raised exception, skip such upload, log the error and keep going with the export instead. Gitlab::ErrorTracking.log_exception(e, project_id: @project.id) end diff --git a/lib/gitlab/jira_import/handle_labels_service.rb b/lib/gitlab/jira_import/handle_labels_service.rb index 1b00515cced..60d7f9e93d9 100644 --- a/lib/gitlab/jira_import/handle_labels_service.rb +++ b/lib/gitlab/jira_import/handle_labels_service.rb @@ -12,7 +12,7 @@ module Gitlab return if jira_labels.blank? existing_labels = LabelsFinder.new(nil, project: project, title: jira_labels) - .execute(skip_authorization: true).select(:id, :name) + .execute(skip_authorization: true).select(:id, :project_id, :group_id, :type, :name) new_labels = create_missing_labels(existing_labels) label_ids = existing_labels.map(&:id) diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 25dffcbe0ee..5057317ae01 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -7,10 +7,6 @@ module Gitlab # see https://jira.atlassian.com/browse/JRACLOUD-67570 # We set it to 1000 in case they change their mind. BATCH_SIZE = 1000 - JIRA_IMPORT_THRESHOLD = 100_000 - JIRA_IMPORT_PAUSE_LIMIT = 50_000 - - RetriesExceededError = Class.new(RuntimeError) attr_reader :imported_items_cache_key, :start_at, :job_waiter @@ -52,7 +48,7 @@ module Gitlab end def schedule_issue_import_workers(issues) - next_iid = project.issues.maximum(:iid).to_i + 1 + next_iid = Issue.with_project_iid_supply(project, &:next_value) issues.each do |jira_issue| # Technically it's possible that the same work is performed multiple @@ -71,13 +67,11 @@ module Gitlab { iid: next_iid } ).execute - # Pause the importer to allow the import to catch up and cache to drain - pause_jira_issue_importer if jira_import_issue_worker.queue_size > JIRA_IMPORT_THRESHOLD - Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key) job_waiter.jobs_remaining += 1 - next_iid += 1 + + next_iid = Issue.with_project_iid_supply(project, &:next_value) # Mark the issue as imported immediately so we don't end up # importing it multiple times within same import. @@ -97,27 +91,6 @@ module Gitlab job_waiter end - def jira_import_issue_worker - @_jira_import_issue_worker ||= Gitlab::JiraImport::ImportIssueWorker - end - - def pause_jira_issue_importer - # Wait for import workers to drop below 50K in the iterations of the timeout - # timeout - Set to 5 seconds. - # Time to process 100K jobs is currently ~14 seconds. - # Source: https://github.com/mperham/sidekiq#performance - # retries - Set to 10 times to avoid indefinitely pause. - # Raises an error if the queue does not reduce below the limit after 10 tries. - - retries = 10 - while retries > 0 && jira_import_issue_worker.queue_size >= JIRA_IMPORT_PAUSE_LIMIT - job_waiter.wait(5) - retries -= 1 - end - - raise RetriesExceededError, 'Retry failed after 10 attempts' if retries == 0 - end - def fetch_issues(start_at) client.Issue.jql("PROJECT='#{jira_project_key}' ORDER BY created ASC", { max_results: BATCH_SIZE, start_at: start_at }) end diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index ce07752f88c..823d6202b1e 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -34,6 +34,7 @@ module Gitlab alias_method :parse!, :parse alias_method :load, :parse + alias_method :decode, :parse # Restricted method for converting a Ruby object to JSON. If you # need to pass options to this, you should use `.generate` instead, @@ -56,6 +57,8 @@ module Gitlab adapter_generate(object, opts) end + alias_method :encode, :generate + # Generates JSON for an object and makes it look purdy # # The Oj variant in this looks seriously weird but these are the settings diff --git a/lib/gitlab/kroki.rb b/lib/gitlab/kroki.rb index fa10e922c80..6799be8e279 100644 --- a/lib/gitlab/kroki.rb +++ b/lib/gitlab/kroki.rb @@ -6,13 +6,13 @@ module Gitlab # Helper methods for Kroki module Kroki BLOCKDIAG_FORMATS = %w[ - blockdiag - seqdiag - actdiag - nwdiag - packetdiag - rackdiag - ].freeze + blockdiag + seqdiag + actdiag + nwdiag + packetdiag + rackdiag + ].freeze DIAGRAMS_FORMATS = (::AsciidoctorExtensions::Kroki::SUPPORTED_DIAGRAM_NAMES - %w(mermaid)).freeze DIAGRAMS_FORMATS_WO_PLANTUML = (DIAGRAMS_FORMATS - %w(plantuml)).freeze diff --git a/lib/gitlab/legacy_github_import/base_formatter.rb b/lib/gitlab/legacy_github_import/base_formatter.rb index 0b19cf742ed..7bb33cd474b 100644 --- a/lib/gitlab/legacy_github_import/base_formatter.rb +++ b/lib/gitlab/legacy_github_import/base_formatter.rb @@ -23,7 +23,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def url - raw_data.url || '' + raw_data[:url] || '' end end end diff --git a/lib/gitlab/legacy_github_import/branch_formatter.rb b/lib/gitlab/legacy_github_import/branch_formatter.rb index 1177751457f..372c6b2e8a0 100644 --- a/lib/gitlab/legacy_github_import/branch_formatter.rb +++ b/lib/gitlab/legacy_github_import/branch_formatter.rb @@ -3,7 +3,17 @@ module Gitlab module LegacyGithubImport class BranchFormatter < BaseFormatter - delegate :repo, :sha, :ref, to: :raw_data + def repo + raw_data[:repo] + end + + def sha + raw_data[:sha] + end + + def ref + raw_data[:ref] + end def exists? branch_exists? && commit_exists? @@ -14,7 +24,7 @@ module Gitlab end def user - raw_data.user&.login || 'unknown' + raw_data.dig(:user, :login) || 'unknown' end def short_sha diff --git a/lib/gitlab/legacy_github_import/comment_formatter.rb b/lib/gitlab/legacy_github_import/comment_formatter.rb index d83cc4f6b3c..ffd9da604ca 100644 --- a/lib/gitlab/legacy_github_import/comment_formatter.rb +++ b/lib/gitlab/legacy_github_import/comment_formatter.rb @@ -9,19 +9,19 @@ module Gitlab { project: project, note: note, - commit_id: raw_data.commit_id, + commit_id: raw_data[:commit_id], line_code: line_code, author_id: author_id, type: type, - created_at: raw_data.created_at, - updated_at: raw_data.updated_at + created_at: raw_data[:created_at], + updated_at: raw_data[:updated_at] } end private def author - @author ||= UserFormatter.new(client, raw_data.user) + @author ||= UserFormatter.new(client, raw_data[:user]) end def author_id @@ -29,7 +29,7 @@ module Gitlab end def body - raw_data.body || "" + raw_data[:body] || "" end def line_code @@ -48,11 +48,11 @@ module Gitlab end def diff_hunk - raw_data.diff_hunk + raw_data[:diff_hunk] end def file_path - raw_data.path + raw_data[:path] end def note diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 4ddafbac4c6..331eab7b62a 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -96,7 +96,7 @@ module Gitlab def import_labels fetch_resources(:labels, repo, per_page: 100) do |labels| labels.each do |raw| - gh_label = LabelFormatter.new(project, raw) + gh_label = LabelFormatter.new(project, raw.to_h) gh_label.create! rescue StandardError => e errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message } @@ -109,7 +109,7 @@ module Gitlab def import_milestones fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| milestones.each do |raw| - gh_milestone = MilestoneFormatter.new(project, raw) + gh_milestone = MilestoneFormatter.new(project, raw.to_h) gh_milestone.create! rescue StandardError => e errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message } @@ -121,6 +121,7 @@ module Gitlab def import_issues fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| issues.each do |raw| + raw = raw.to_h gh_issue = IssueFormatter.new(project, raw, client) begin @@ -143,6 +144,7 @@ module Gitlab def import_pull_requests fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| pull_requests.each do |raw| + raw = raw.to_h gh_pull_request = PullRequestFormatter.new(project, raw, client) next unless gh_pull_request.valid? @@ -190,10 +192,12 @@ module Gitlab end def apply_labels(issuable, raw) - return unless raw.labels.count > 0 + raw = raw.to_h - label_ids = raw.labels - .map { |attrs| @labels[attrs.name] } + return unless raw[:labels].count > 0 + + label_ids = raw[:labels] + .map { |attrs| @labels[attrs[:name]] } .compact issuable.update_attribute(:label_ids, label_ids) @@ -226,10 +230,12 @@ module Gitlab def create_comments(comments) ActiveRecord::Base.no_touching do comments.each do |raw| + raw = raw.to_h + comment = CommentFormatter.new(project, raw, client) # GH does not return info about comment's parent, so we guess it by checking its URL! - *_, parent, iid = URI(raw.html_url).path.split('/') + *_, parent, iid = URI(raw[:html_url]).path.split('/') issuable = if parent == 'issues' Issue.find_by(project_id: project.id, iid: iid) @@ -241,7 +247,7 @@ module Gitlab issuable.notes.create!(comment.attributes) rescue StandardError => e - errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw[:url]), errors: e.message } end end end @@ -251,7 +257,7 @@ module Gitlab last_note_attrs = nil cut_off_index = comments.find_index do |raw| - comment = CommentFormatter.new(project, raw) + comment = CommentFormatter.new(project, raw.to_h) comment_attrs = comment.attributes last_note_attrs ||= last_note.slice(*comment_attrs.keys) @@ -282,7 +288,7 @@ module Gitlab def import_releases fetch_resources(:releases, repo, per_page: 100) do |releases| releases.each do |raw| - gh_release = ReleaseFormatter.new(project, raw) + gh_release = ReleaseFormatter.new(project, raw.to_h) gh_release.create! if gh_release.valid? rescue StandardError => e errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message } diff --git a/lib/gitlab/legacy_github_import/issuable_formatter.rb b/lib/gitlab/legacy_github_import/issuable_formatter.rb index 1a0aefbbd62..e4e333735be 100644 --- a/lib/gitlab/legacy_github_import/issuable_formatter.rb +++ b/lib/gitlab/legacy_github_import/issuable_formatter.rb @@ -9,7 +9,9 @@ module Gitlab raise NotImplementedError end - delegate :number, to: :raw_data + def number + raw_data[:number] + end def find_condition { iid: number } @@ -18,15 +20,15 @@ module Gitlab private def state - raw_data.state == 'closed' ? 'closed' : 'opened' + raw_data[:state] == 'closed' ? 'closed' : 'opened' end def assigned? - raw_data.assignee.present? + raw_data[:assignee].present? end def author - @author ||= UserFormatter.new(client, raw_data.user) + @author ||= UserFormatter.new(client, raw_data[:user]) end def author_id @@ -35,7 +37,7 @@ module Gitlab def assignee if assigned? - @assignee ||= UserFormatter.new(client, raw_data.assignee) + @assignee ||= UserFormatter.new(client, raw_data[:assignee]) end end @@ -46,7 +48,7 @@ module Gitlab end def body - raw_data.body || "" + raw_data[:body] || "" end def description @@ -59,8 +61,8 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def milestone - if raw_data.milestone.present? - milestone = MilestoneFormatter.new(project, raw_data.milestone) + if raw_data[:milestone].present? + milestone = MilestoneFormatter.new(project, raw_data[:milestone]) project.milestones.find_by(milestone.find_condition) end end diff --git a/lib/gitlab/legacy_github_import/issue_formatter.rb b/lib/gitlab/legacy_github_import/issue_formatter.rb index 2f46e2e30d1..e5c568207e3 100644 --- a/lib/gitlab/legacy_github_import/issue_formatter.rb +++ b/lib/gitlab/legacy_github_import/issue_formatter.rb @@ -8,18 +8,18 @@ module Gitlab iid: number, project: project, milestone: milestone, - title: raw_data.title, + title: raw_data[:title], description: description, state: state, author_id: author_id, assignee_ids: Array(assignee_id), - created_at: raw_data.created_at, - updated_at: raw_data.updated_at + created_at: raw_data[:created_at], + updated_at: raw_data[:updated_at] } end def has_comments? - raw_data.comments > 0 + raw_data[:comments] > 0 end def project_association @@ -27,7 +27,7 @@ module Gitlab end def pull_request? - raw_data.pull_request.present? + raw_data[:pull_request].present? end end end diff --git a/lib/gitlab/legacy_github_import/label_formatter.rb b/lib/gitlab/legacy_github_import/label_formatter.rb index 415b1b8878f..e3b767f41fa 100644 --- a/lib/gitlab/legacy_github_import/label_formatter.rb +++ b/lib/gitlab/legacy_github_import/label_formatter.rb @@ -28,11 +28,11 @@ module Gitlab private def color - "##{raw_data.color}" + "##{raw_data[:color]}" end def title - raw_data.name + raw_data[:name] end end end diff --git a/lib/gitlab/legacy_github_import/milestone_formatter.rb b/lib/gitlab/legacy_github_import/milestone_formatter.rb index 2fe1b4258d3..60d5bcbf44a 100644 --- a/lib/gitlab/legacy_github_import/milestone_formatter.rb +++ b/lib/gitlab/legacy_github_import/milestone_formatter.rb @@ -7,12 +7,12 @@ module Gitlab { iid: number, project: project, - title: raw_data.title, - description: raw_data.description, - due_date: raw_data.due_on, + title: raw_data[:title], + description: raw_data[:description], + due_date: raw_data[:due_on], state: state, - created_at: raw_data.created_at, - updated_at: raw_data.updated_at + created_at: raw_data[:created_at], + updated_at: raw_data[:updated_at] } end @@ -26,16 +26,16 @@ module Gitlab def number if project.gitea_import? - raw_data.id + raw_data[:id] else - raw_data.number + raw_data[:number] end end private def state - raw_data.state == 'closed' ? 'closed' : 'active' + raw_data[:state] == 'closed' ? 'closed' : 'active' end end end diff --git a/lib/gitlab/legacy_github_import/pull_request_formatter.rb b/lib/gitlab/legacy_github_import/pull_request_formatter.rb index 5b847f13d4a..72adba30093 100644 --- a/lib/gitlab/legacy_github_import/pull_request_formatter.rb +++ b/lib/gitlab/legacy_github_import/pull_request_formatter.rb @@ -9,7 +9,7 @@ module Gitlab def attributes { iid: number, - title: raw_data.title, + title: raw_data[:title], description: description, source_project: source_branch_project, source_branch: source_branch_name, @@ -21,8 +21,8 @@ module Gitlab milestone: milestone, author_id: author_id, assignee_id: assignee_id, - created_at: raw_data.created_at, - updated_at: raw_data.updated_at, + created_at: raw_data[:created_at], + updated_at: raw_data[:updated_at], imported: true } end @@ -36,7 +36,7 @@ module Gitlab end def source_branch - @source_branch ||= BranchFormatter.new(project, raw_data.head) + @source_branch ||= BranchFormatter.new(project, raw_data[:head]) end def source_branch_name @@ -57,7 +57,7 @@ module Gitlab end def target_branch - @target_branch ||= BranchFormatter.new(project, raw_data.base) + @target_branch ||= BranchFormatter.new(project, raw_data[:base]) end def target_branch_name @@ -71,7 +71,7 @@ module Gitlab def cross_project? return true if source_branch_repo.nil? - source_branch_repo.id != target_branch_repo.id + source_branch_repo[:id] != target_branch_repo[:id] end def opened? @@ -81,7 +81,7 @@ module Gitlab private def state - if raw_data.state == 'closed' && raw_data.merged_at.present? + if raw_data[:state] == 'closed' && raw_data[:merged_at].present? 'merged' else super diff --git a/lib/gitlab/legacy_github_import/release_formatter.rb b/lib/gitlab/legacy_github_import/release_formatter.rb index 0fb7e376f5b..2a54a15429b 100644 --- a/lib/gitlab/legacy_github_import/release_formatter.rb +++ b/lib/gitlab/legacy_github_import/release_formatter.rb @@ -6,13 +6,13 @@ module Gitlab def attributes { project: project, - tag: raw_data.tag_name, - name: raw_data.name, - description: raw_data.body, - created_at: raw_data.created_at, + tag: raw_data[:tag_name], + name: raw_data[:name], + description: raw_data[:body], + created_at: raw_data[:created_at], # Draft releases will have a null published_at - released_at: raw_data.published_at || Time.current, - updated_at: raw_data.created_at + released_at: raw_data[:published_at] || Time.current, + updated_at: raw_data[:created_at] } end @@ -21,11 +21,11 @@ module Gitlab end def find_condition - { tag: raw_data.tag_name } + { tag: raw_data[:tag_name] } end def valid? - !raw_data.draft && raw_data.tag_name.present? + !raw_data[:draft] && raw_data[:tag_name].present? end end end diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index 7ae1b195ec6..d45a166d2b7 100644 --- a/lib/gitlab/legacy_github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -5,13 +5,19 @@ module Gitlab class UserFormatter attr_reader :client, :raw - delegate :id, :login, to: :raw, allow_nil: true - def initialize(client, raw) @client = client @raw = raw end + def id + raw[:id] + end + + def login + raw[:login] + end + def gitlab_id return @gitlab_id if defined?(@gitlab_id) @@ -21,7 +27,7 @@ module Gitlab private def email - @email ||= client.user(raw.login).try(:email) + @email ||= client.user(raw[:login]).to_h[:email] end def find_by_email diff --git a/lib/gitlab/memory/diagnostic_reports_logger.rb b/lib/gitlab/memory/diagnostic_reports_logger.rb new file mode 100644 index 00000000000..cc5b719fa19 --- /dev/null +++ b/lib/gitlab/memory/diagnostic_reports_logger.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'logger' + +module Gitlab + module Memory + class DiagnosticReportsLogger < ::Logger + def format_message(severity, timestamp, progname, message) + data = {} + data[:severity] = severity + data[:time] = timestamp.utc.iso8601(3) + + data.merge!(message) + + "#{JSON.generate(data)}\n" # rubocop:disable Gitlab/Json + end + end + end +end diff --git a/lib/gitlab/memory/reports_daemon.rb b/lib/gitlab/memory/reports_daemon.rb index ed1da8baab5..0dfc31235e7 100644 --- a/lib/gitlab/memory/reports_daemon.rb +++ b/lib/gitlab/memory/reports_daemon.rb @@ -7,7 +7,7 @@ module Gitlab DEFAULT_SLEEP_MAX_DELTA_S = 600 # 0..10 minutes DEFAULT_SLEEP_BETWEEN_REPORTS_S = 120 # 2 minutes - DEFAULT_REPORTS_PATH = '/tmp' + DEFAULT_REPORTS_PATH = Dir.tmpdir def initialize(**options) super diff --git a/lib/gitlab/memory/reports_uploader.rb b/lib/gitlab/memory/reports_uploader.rb new file mode 100644 index 00000000000..76c3e0862e2 --- /dev/null +++ b/lib/gitlab/memory/reports_uploader.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require_relative '../metrics/system' + +module Gitlab + module Memory + class ReportsUploader + def initialize(gcs_key:, gcs_project:, gcs_bucket:, logger:) + @gcs_bucket = gcs_bucket + @fog = Fog::Storage::Google.new(google_project: gcs_project, google_json_key_location: gcs_key) + @logger = logger + end + + def upload(path) + log_upload_requested(path) + start_monotonic_time = Gitlab::Metrics::System.monotonic_time + + File.open(path.to_s) { |file| fog.put_object(gcs_bucket, File.basename(path), file) } + + duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time + log_upload_success(path, duration_s) + rescue StandardError, Errno::ENOENT => error + log_exception(error) + end + + private + + attr_reader :gcs_bucket, :fog, :logger + + def log_upload_requested(path) + logger.info(log_labels.merge(perf_report_status: 'upload requested', perf_report_path: path)) + end + + def log_upload_success(path, duration_s) + logger.info(log_labels.merge(perf_report_status: 'upload success', perf_report_path: path, + duration_s: duration_s)) + end + + def log_exception(error) + logger.error(log_labels.merge(perf_report_status: "error", error: error.message)) + end + + def log_labels + { + message: "Diagnostic reports", + class: self.class.name, + pid: $$ + } + end + end + end +end diff --git a/lib/gitlab/memory/upload_and_cleanup_reports.rb b/lib/gitlab/memory/upload_and_cleanup_reports.rb new file mode 100644 index 00000000000..27d94df478c --- /dev/null +++ b/lib/gitlab/memory/upload_and_cleanup_reports.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class UploadAndCleanupReports + DEFAULT_SLEEP_TIME_SECONDS = 900 # 15 minutes + + def initialize( + uploader:, + reports_path:, + logger:, + sleep_time_seconds: ENV['GITLAB_DIAGNOSTIC_REPORTS_UPLOADER_SLEEP_S']&.to_i || DEFAULT_SLEEP_TIME_SECONDS) + + @uploader = uploader + @reports_path = reports_path + @sleep_time_seconds = sleep_time_seconds + @alive = true + @logger = logger + end + + attr_reader :uploader, :reports_path, :sleep_time_seconds, :logger + + def call + log_started + + loop do + sleep(sleep_time_seconds) + + files_to_process.each { |path| upload_and_cleanup!(path) } + end + end + + private + + def upload_and_cleanup!(path) + uploader.upload(path) + rescue StandardError, Errno::ENOENT => error + log_exception(error) + ensure + cleanup!(path) + end + + def cleanup!(path) + File.unlink(path) if File.exist?(path) + rescue Errno::ENOENT + # Path does not exist: Ignore. We already check `File.exist?`. Rescue to be extra safe. + end + + def files_to_process + Dir.entries(reports_path) + .map { |path| File.join(reports_path, path) } + .select { |path| File.file?(path) } + end + + def log_started + logger.info(log_labels.merge(perf_report_status: "started")) + end + + def log_exception(error) + logger.error(log_labels.merge(perf_report_status: "error", error: error.message)) + end + + def log_labels + { + message: "Diagnostic reports", + class: self.class.name, + pid: $$ + } + end + end + end +end diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index 38231fa933b..7007fdfe386 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -2,25 +2,10 @@ module Gitlab module Memory - # A background thread that observes Ruby heap fragmentation and calls - # into a handler when the Ruby heap has been fragmented for an extended - # period of time. - # - # See Gitlab::Metrics::Memory for how heap fragmentation is defined. - # - # To decide whether a given fragmentation level is being exceeded, - # the watchdog regularly polls the GC. Whenever a violation occurs - # a strike is issued. If the maximum number of strikes are reached, - # a handler is invoked to deal with the situation. - # - # The duration for which a process may be above a given fragmentation - # threshold is computed as `max_strikes * sleep_time_seconds`. + # A background thread that monitors Ruby memory and calls + # into a handler when the Ruby process violates defined limits + # for an extended period of time. class Watchdog - DEFAULT_SLEEP_TIME_SECONDS = 60 * 5 - DEFAULT_MAX_HEAP_FRAG = 0.5 - DEFAULT_MAX_MEM_GROWTH = 3.0 - DEFAULT_MAX_STRIKES = 5 - # This handler does nothing. It returns `false` to indicate to the # caller that the situation has not been dealt with so it will # receive calls repeatedly if fragmentation remains high. @@ -62,73 +47,27 @@ module Gitlab end end - # max_heap_fragmentation: - # The degree to which the Ruby heap is allowed to be fragmented. Range [0,1]. - # max_mem_growth: - # A multiplier for how much excess private memory a worker can map compared to a reference process - # (itself or the primary in a pre-fork server.) - # max_strikes: - # How many times the process is allowed to be above max_heap_fragmentation before - # a handler is invoked. - # sleep_time_seconds: - # Used to control the frequency with which the watchdog will wake up and poll the GC. - def initialize( - handler: NullHandler.instance, - logger: Logger.new($stdout), - max_heap_fragmentation: ENV['GITLAB_MEMWD_MAX_HEAP_FRAG']&.to_f || DEFAULT_MAX_HEAP_FRAG, - max_mem_growth: ENV['GITLAB_MEMWD_MAX_MEM_GROWTH']&.to_f || DEFAULT_MAX_MEM_GROWTH, - max_strikes: ENV['GITLAB_MEMWD_MAX_STRIKES']&.to_i || DEFAULT_MAX_STRIKES, - sleep_time_seconds: ENV['GITLAB_MEMWD_SLEEP_TIME_SEC']&.to_i || DEFAULT_SLEEP_TIME_SECONDS, - **options) - super(**options) - - @handler = handler - @logger = logger - @sleep_time_seconds = sleep_time_seconds - @max_strikes = max_strikes - @stats = { - heap_frag: { - max: max_heap_fragmentation, - strikes: 0 - }, - mem_growth: { - max: max_mem_growth, - strikes: 0 - } - } - + def initialize + @configuration = Configuration.new @alive = true - init_prometheus_metrics(max_heap_fragmentation) - end - - attr_reader :max_strikes, :sleep_time_seconds - - def max_heap_fragmentation - @stats[:heap_frag][:max] - end - - def max_mem_growth - @stats[:mem_growth][:max] + init_prometheus_metrics end - def strikes(stat) - @stats[stat][:strikes] + def configure + yield @configuration end def call - @logger.info(log_labels.merge(message: 'started')) + logger.info(log_labels.merge(message: 'started')) while @alive - sleep(@sleep_time_seconds) - - next unless Feature.enabled?(:gitlab_memory_watchdog, type: :ops) + sleep(sleep_time_seconds) - monitor_heap_fragmentation - monitor_memory_growth + monitor if Feature.enabled?(:gitlab_memory_watchdog, type: :ops) end - @logger.info(log_labels.merge(message: 'stopped')) + logger.info(log_labels.merge(message: 'stopped')) end def stop @@ -137,71 +76,24 @@ module Gitlab private - def monitor_memory_condition(stat_key) - return unless @alive - - stat = @stats[stat_key] - - ok, labels = yield(stat) + def monitor + @configuration.monitors.call_each do |result| + break unless @alive - if ok - stat[:strikes] = 0 - else - stat[:strikes] += 1 - @counter_violations.increment(reason: stat_key.to_s) - end + next unless result.threshold_violated? - if stat[:strikes] > @max_strikes - @alive = !memory_limit_exceeded_callback(stat_key, labels) - stat[:strikes] = 0 - end - end + @counter_violations.increment(reason: result.monitor_name) - def monitor_heap_fragmentation - monitor_memory_condition(:heap_frag) do |stat| - heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation - [ - heap_fragmentation <= stat[:max], - { - message: 'heap fragmentation limit exceeded', - memwd_cur_heap_frag: heap_fragmentation, - memwd_max_heap_frag: stat[:max] - } - ] - end - end + next unless result.strikes_exceeded? - def monitor_memory_growth - monitor_memory_condition(:mem_growth) do |stat| - worker_uss = Gitlab::Metrics::System.memory_usage_uss_pss[:uss] - reference_uss = reference_mem[:uss] - memory_limit = stat[:max] * reference_uss - [ - worker_uss <= memory_limit, - { - message: 'memory limit exceeded', - memwd_uss_bytes: worker_uss, - memwd_ref_uss_bytes: reference_uss, - memwd_max_uss_bytes: memory_limit - } - ] + @alive = !memory_limit_exceeded_callback(result.monitor_name, result.payload) end end - # On pre-fork systems this would be the primary process memory from which workers fork. - # Otherwise it is the current process' memory. - # - # We initialize this lazily because in the initializer the application may not have - # finished booting yet, which would yield an incorrect baseline. - def reference_mem - @reference_mem ||= Gitlab::Metrics::System.memory_usage_uss_pss(pid: Gitlab::Cluster::PRIMARY_PID) - end - - def memory_limit_exceeded_callback(stat_key, handler_labels) - all_labels = log_labels.merge(handler_labels) - .merge(memwd_cur_strikes: strikes(stat_key)) - @logger.warn(all_labels) - @counter_violations_handled.increment(reason: stat_key.to_s) + def memory_limit_exceeded_callback(monitor_name, monitor_payload) + all_labels = log_labels.merge(monitor_payload) + logger.warn(all_labels) + @counter_violations_handled.increment(reason: monitor_name) handler.call end @@ -211,7 +103,15 @@ module Gitlab # all that happens is we collect logs and Prometheus events for fragmentation violations. return NullHandler.instance unless Feature.enabled?(:enforce_memory_watchdog, type: :ops) - @handler + @configuration.handler + end + + def logger + @configuration.logger + end + + def sleep_time_seconds + @configuration.sleep_time_seconds end def log_labels @@ -219,27 +119,20 @@ module Gitlab pid: $$, worker_id: worker_id, memwd_handler_class: handler.class.name, - memwd_sleep_time_s: @sleep_time_seconds, - memwd_max_strikes: @max_strikes, + memwd_sleep_time_s: sleep_time_seconds, memwd_rss_bytes: process_rss_bytes } end - def worker_id - ::Prometheus::PidProvider.worker_id - end - def process_rss_bytes Gitlab::Metrics::System.memory_usage_rss end - def init_prometheus_metrics(max_heap_fragmentation) - @heap_frag_limit = Gitlab::Metrics.gauge( - :gitlab_memwd_heap_frag_limit, - 'The configured limit for how fragmented the Ruby heap is allowed to be' - ) - @heap_frag_limit.set({}, max_heap_fragmentation) + def worker_id + ::Prometheus::PidProvider.worker_id + end + def init_prometheus_metrics default_labels = { pid: worker_id } @counter_violations = Gitlab::Metrics.counter( :gitlab_memwd_violations_total, diff --git a/lib/gitlab/memory/watchdog/configuration.rb b/lib/gitlab/memory/watchdog/configuration.rb new file mode 100644 index 00000000000..2d84b083f55 --- /dev/null +++ b/lib/gitlab/memory/watchdog/configuration.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + class Configuration + class MonitorStack + def initialize + @monitors = [] + end + + def use(monitor_class, *args, **kwargs, &block) + remove(monitor_class) + @monitors.push(build_monitor_state(monitor_class, *args, **kwargs, &block)) + end + + def call_each + @monitors.each do |monitor| + yield monitor.call + end + end + + private + + def remove(monitor_class) + @monitors.delete_if { |monitor| monitor.monitor_class == monitor_class } + end + + def build_monitor_state(monitor_class, *args, max_strikes:, **kwargs, &block) + monitor = build_monitor(monitor_class, *args, **kwargs, &block) + + Gitlab::Memory::Watchdog::MonitorState.new(monitor, max_strikes: max_strikes) + end + + def build_monitor(monitor_class, *args, **kwargs, &block) + monitor_class.new(*args, **kwargs, &block) + end + end + + DEFAULT_SLEEP_TIME_SECONDS = 60 + + attr_reader :monitors + attr_writer :logger, :handler, :sleep_time_seconds + + def initialize + @monitors = MonitorStack.new + end + + def handler + @handler ||= NullHandler.instance + end + + def logger + @logger ||= Gitlab::Logger.new($stdout) + end + + # Used to control the frequency with which the watchdog will wake up and poll the GC. + def sleep_time_seconds + @sleep_time_seconds ||= DEFAULT_SLEEP_TIME_SECONDS + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb b/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb new file mode 100644 index 00000000000..7748c19c6d8 --- /dev/null +++ b/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + module Monitor + # A monitor that observes Ruby heap fragmentation and calls + # memory_violation_callback when the Ruby heap has been fragmented for an extended + # period of time. + # + # See Gitlab::Metrics::Memory for how heap fragmentation is defined. + class HeapFragmentation + attr_reader :max_heap_fragmentation + + # max_heap_fragmentation: + # The degree to which the Ruby heap is allowed to be fragmented. Range [0,1]. + def initialize(max_heap_fragmentation:) + @max_heap_fragmentation = max_heap_fragmentation + init_frag_limit_metrics + end + + def call + heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation + + return { threshold_violated: false, payload: {} } unless heap_fragmentation > max_heap_fragmentation + + { threshold_violated: true, payload: payload(heap_fragmentation) } + end + + private + + def payload(heap_fragmentation) + { + message: 'heap fragmentation limit exceeded', + memwd_cur_heap_frag: heap_fragmentation, + memwd_max_heap_frag: max_heap_fragmentation + } + end + + def init_frag_limit_metrics + heap_frag_limit = Gitlab::Metrics.gauge( + :gitlab_memwd_heap_frag_limit, + 'The configured limit for how fragmented the Ruby heap is allowed to be' + ) + heap_frag_limit.set({}, max_heap_fragmentation) + end + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/monitor/unique_memory_growth.rb b/lib/gitlab/memory/watchdog/monitor/unique_memory_growth.rb new file mode 100644 index 00000000000..2a1512c4cff --- /dev/null +++ b/lib/gitlab/memory/watchdog/monitor/unique_memory_growth.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + module Monitor + class UniqueMemoryGrowth + attr_reader :max_mem_growth + + def initialize(max_mem_growth:) + @max_mem_growth = max_mem_growth + end + + def call + worker_uss = Gitlab::Metrics::System.memory_usage_uss_pss[:uss] + reference_uss = reference_mem[:uss] + memory_limit = max_mem_growth * reference_uss + + return { threshold_violated: false, payload: {} } unless worker_uss > memory_limit + + { threshold_violated: true, payload: payload(worker_uss, reference_uss, memory_limit) } + end + + private + + def payload(worker_uss, reference_uss, memory_limit) + { + message: 'memory limit exceeded', + memwd_uss_bytes: worker_uss, + memwd_ref_uss_bytes: reference_uss, + memwd_max_uss_bytes: memory_limit + } + end + + # On pre-fork systems this would be the primary process memory from which workers fork. + # Otherwise it is the current process' memory. + # + # We initialize this lazily because in the initializer the application may not have + # finished booting yet, which would yield an incorrect baseline. + def reference_mem + @reference_mem ||= Gitlab::Metrics::System.memory_usage_uss_pss(pid: Gitlab::Cluster::PRIMARY_PID) + end + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/monitor_state.rb b/lib/gitlab/memory/watchdog/monitor_state.rb new file mode 100644 index 00000000000..73be5de3e45 --- /dev/null +++ b/lib/gitlab/memory/watchdog/monitor_state.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + class MonitorState + class Result + attr_reader :payload + + def initialize(strikes_exceeded:, threshold_violated:, monitor_class:, payload: ) + @strikes_exceeded = strikes_exceeded + @threshold_violated = threshold_violated + @monitor_class = monitor_class + @payload = payload + end + + def strikes_exceeded? + @strikes_exceeded + end + + def threshold_violated? + @threshold_violated + end + + def monitor_name + @monitor_class.name.demodulize.underscore.to_sym + end + end + + def initialize(monitor, max_strikes:) + @monitor = monitor + @max_strikes = max_strikes + @strikes = 0 + end + + def call + reset_strikes if strikes_exceeded? + + monitor_result = @monitor.call + + if monitor_result[:threshold_violated] + issue_strike + else + reset_strikes + end + + build_result(monitor_result) + end + + def monitor_class + @monitor.class + end + + private + + def build_result(monitor_result) + Result.new( + strikes_exceeded: strikes_exceeded?, + monitor_class: monitor_class, + threshold_violated: monitor_result[:threshold_violated], + payload: payload.merge(monitor_result[:payload])) + end + + def payload + { + memwd_max_strikes: @max_strikes, + memwd_cur_strikes: @strikes + } + end + + def strikes_exceeded? + @strikes > @max_strikes + end + + def issue_strike + @strikes += 1 + end + + def reset_strikes + @strikes = 0 + end + end + end + end +end diff --git a/lib/gitlab/metrics/global_search_slis.rb b/lib/gitlab/metrics/global_search_slis.rb index e37129fed38..3400a6c78ef 100644 --- a/lib/gitlab/metrics/global_search_slis.rb +++ b/lib/gitlab/metrics/global_search_slis.rb @@ -13,9 +13,7 @@ module Gitlab ADVANCED_CODE_TARGET_S = 13.546 def initialize_slis! - if Feature.enabled?(:global_search_custom_slis) - Gitlab::Metrics::Sli::Apdex.initialize_sli(:global_search, possible_labels) - end + Gitlab::Metrics::Sli::Apdex.initialize_sli(:global_search, possible_labels) return unless Feature.enabled?(:global_search_error_rate_sli) @@ -23,8 +21,6 @@ module Gitlab end def record_apdex(elapsed:, search_type:, search_level:, search_scope:) - return unless Feature.enabled?(:global_search_custom_slis) - Gitlab::Metrics::Sli::Apdex[:global_search].increment( labels: labels(search_type: search_type, search_level: search_level, search_scope: search_scope), success: elapsed < duration_target(search_type, search_scope) diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index d7eef722d6e..affadc4274c 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -14,11 +14,13 @@ module Gitlab PROC_SMAPS_ROLLUP_PATH = '/proc/%s/smaps_rollup' PROC_LIMITS_PATH = '/proc/self/limits' PROC_FD_GLOB = '/proc/self/fd/*' + PROC_MEM_INFO = '/proc/meminfo' PRIVATE_PAGES_PATTERN = /^(Private_Clean|Private_Dirty|Private_Hugetlb):\s+(?<value>\d+)/.freeze PSS_PATTERN = /^Pss:\s+(?<value>\d+)/.freeze RSS_PATTERN = /VmRSS:\s+(?<value>\d+)/.freeze MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/.freeze + MEM_TOTAL_PATTERN = /^MemTotal:\s+(?<value>\d+) (.+)/.freeze def summary proportional_mem = memory_usage_uss_pss @@ -45,6 +47,10 @@ module Gitlab .transform_values(&:kilobytes) end + def memory_total + sum_matches(PROC_MEM_INFO, memory_total: MEM_TOTAL_PATTERN)[:memory_total].kilobytes + end + def file_descriptor_count Dir.glob(PROC_FD_GLOB).length end diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 0576aed811c..12d4d9d8928 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -138,9 +138,9 @@ module Gitlab .with .recursive(ancestors.to_arel, descendants.to_arel) .from_union([ - ancestors_scope, - descendants_scope - ]) + ancestors_scope, + descendants_scope + ]) read_only(relation) end diff --git a/lib/gitlab/pages/cache_control.rb b/lib/gitlab/pages/cache_control.rb index 991a1297d03..187d5f907e4 100644 --- a/lib/gitlab/pages/cache_control.rb +++ b/lib/gitlab/pages/cache_control.rb @@ -3,9 +3,13 @@ module Gitlab module Pages class CacheControl - CACHE_KEY_FORMAT = 'pages_domain_for_%{type}_%{id}' + include Gitlab::Utils::StrongMemoize - attr_reader :cache_key + EXPIRE = 12.hours + # To avoid delivering expired deployment URL in the cached payload, + # use a longer expiration time in the deployment URL + DEPLOYMENT_EXPIRATION = (EXPIRE + 12.hours) + CACHE_KEY_FORMAT = 'pages_domain_for_%{type}_%{id}_%{settings}' class << self def for_project(project_id) @@ -20,12 +24,35 @@ module Gitlab def initialize(type:, id:) raise(ArgumentError, "type must be :namespace or :project") unless %i[namespace project].include?(type) - @cache_key = CACHE_KEY_FORMAT % { type: type, id: id } + @type = type + @id = id + end + + def cache_key + strong_memoize(:cache_key) do + CACHE_KEY_FORMAT % { + type: @type, + id: @id, + settings: settings + } + end end def clear_cache Rails.cache.delete(cache_key) end + + private + + def settings + values = ::Gitlab.config.pages.dup + + values['app_settings'] = ::Gitlab::CurrentSettings.attributes.slice( + 'force_pages_access_control' + ) + + ::Digest::SHA256.hexdigest(values.inspect) + end end end end diff --git a/lib/gitlab/pagination/keyset/simple_order_builder.rb b/lib/gitlab/pagination/keyset/simple_order_builder.rb index c36bd497aa3..318720c77d1 100644 --- a/lib/gitlab/pagination/keyset/simple_order_builder.rb +++ b/lib/gitlab/pagination/keyset/simple_order_builder.rb @@ -129,28 +129,31 @@ module Gitlab end def primary_key_descending_order - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: model_class.primary_key, - order_expression: arel_table[primary_key].desc - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: model_class.primary_key, + order_expression: arel_table[primary_key].desc + ) + ]) end def primary_key_order - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: model_class.primary_key, - order_expression: order_values.first - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: model_class.primary_key, + order_expression: order_values.first + ) + ]) end def column_with_tie_breaker_order(tie_breaker_column_order = default_tie_breaker_column_order) - Gitlab::Pagination::Keyset::Order.build([ - column(order_values.first), - tie_breaker_column_order - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + column(order_values.first), + tie_breaker_column_order + ]) end def column(order_value) diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_cron_poller.rb deleted file mode 100644 index 630c364d455..00000000000 --- a/lib/gitlab/patch/sidekiq_cron_poller.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -# Patch to address https://github.com/ondrejbartas/sidekiq-cron/issues/361 -# This restores the poll interval to v1.2.0 behavior -# https://github.com/ondrejbartas/sidekiq-cron/blob/v1.2.0/lib/sidekiq/cron/poller.rb#L36-L38 -# This patch only applies to v1.4.0 -require 'sidekiq/cron/version' - -if Gem::Version.new(Sidekiq::Cron::VERSION) != Gem::Version.new('1.4.0') - raise 'New version of sidekiq-cron detected, please remove or update this patch' -end - -module Gitlab - module Patch - module SidekiqCronPoller - def poll_interval_average - Sidekiq.options[:poll_interval] || Sidekiq::Cron::POLL_INTERVAL - end - end - end -end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index fd9f73d18c1..f8a85f693bc 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -43,12 +43,9 @@ module Gitlab # - private_token: instead of providing a user instance, the token can be # given as a string. Takes precedence over the user option. # - # - sampling_mode: When true, uses a sampling profiler (StackProf) instead of a tracing profiler (RubyProf). - # - # - profiler_options: A keyword Hash of arguments passed to the profiler. Defaults by profiler type: - # RubyProf - {} - # StackProf - { mode: :wall, out: <some temporary file>, interval: 1000, raw: true } - def self.profile(url, logger: nil, post_data: nil, user: nil, private_token: nil, sampling_mode: false, profiler_options: {}) + # - profiler_options: A keyword Hash of arguments passed to the profiler. Defaults: + # { mode: :wall, out: <some temporary file>, interval: 1000, raw: true } + def self.profile(url, logger: nil, post_data: nil, user: nil, private_token: nil, profiler_options: {}) app = ActionDispatch::Integration::Session.new(Rails.application) verb = :get headers = {} @@ -80,7 +77,7 @@ module Gitlab with_custom_logger(logger) do with_user(user) do - with_profiler(sampling_mode, profiler_options) do + with_profiler(profiler_options) do app.public_send(verb, url, params: post_data, headers: headers) # rubocop:disable GitlabSecurity/PublicSend end end @@ -174,21 +171,11 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def self.print_by_total_time(result, options = {}) - default_options = { sort_method: :total_time, filter_by: :total_time } - - RubyProf::FlatPrinter.new(result).print($stdout, default_options.merge(options)) - end - - def self.with_profiler(sampling_mode, profiler_options) - if sampling_mode - require 'stackprof' - args = { mode: :wall, interval: 1000, raw: true }.merge!(profiler_options) - args[:out] ||= ::Tempfile.new(["profile-#{Time.now.to_i}-", ".dump"]).path - ::StackProf.run(**args) { yield } - else - RubyProf.profile(**profiler_options) { yield } - end + def self.with_profiler(profiler_options) + require 'stackprof' + args = { mode: :wall, interval: 1000, raw: true }.merge!(profiler_options) + args[:out] ||= ::Tempfile.new(["profile-#{Time.now.to_i}-", ".dump"]).path + ::StackProf.run(**args) { yield } end end end diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb index 1d7b179baf0..da3f67dde51 100644 --- a/lib/gitlab/project_authorizations.rb +++ b/lib/gitlab/project_authorizations.rb @@ -37,9 +37,9 @@ module Gitlab Namespace .unscoped .select([ - links[:project_id], - least(cte_alias[:access_level], links[:group_access], 'access_level') - ]) + links[:project_id], + least(cte_alias[:access_level], links[:group_access], 'access_level') + ]) .from(cte_alias) .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') @@ -79,9 +79,9 @@ module Gitlab # Sub groups of any groups the user is a member of. cte << Group.select([ - namespaces[:id], - greatest(members[:access_level], cte.table[:access_level], 'access_level') - ]) + namespaces[:id], + greatest(members[:access_level], cte.table[:access_level], 'access_level') + ]) .joins(join_cte(cte)) .joins(join_members_on_namespaces) .except(:order) diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index 2e31849caaa..46c0a0ddf7a 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -14,8 +14,13 @@ module Gitlab # The maximum number of SQL queries that can be executed in a request. For # the sake of keeping things simple we hardcode this value here, it's not # supposed to be changed very often anyway. - THRESHOLD = 100 - LOG_THRESHOLD = THRESHOLD * 1.5 + def self.threshold + 100 + end + + def self.log_threshold + threshold * 1.5 + end # Error that is raised whenever exceeding the maximum number of queries. ThresholdExceededError = Class.new(StandardError) @@ -76,7 +81,7 @@ module Gitlab end def executed_sql(sql) - return if @count > LOG_THRESHOLD || ignorable?(sql) + return if @count > self.class.log_threshold || ignorable?(sql) @sql_executed << sql end @@ -86,15 +91,15 @@ module Gitlab end def threshold_exceeded? - count > THRESHOLD + count > self.class.threshold end def error_message header = 'Too many SQL queries were executed' header = "#{header} in #{action}" if action - msg = "a maximum of #{THRESHOLD} is allowed but #{count} SQL queries were executed" + msg = "a maximum of #{self.class.threshold} is allowed but #{count} SQL queries were executed" log = @sql_executed.each_with_index.map { |sql, i| "#{i}: #{sql}" }.join("\n").presence - ellipsis = '...' if @count > LOG_THRESHOLD + ellipsis = '...' if @count > self.class.log_threshold ["#{header}: #{msg}", log, ellipsis].compact.join("\n") end @@ -105,3 +110,5 @@ module Gitlab end end end + +Gitlab::QueryLimiting::Transaction.prepend_mod diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index d38b81bff0b..f0ad6653c5e 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -176,7 +176,7 @@ module Gitlab explanation { _('Approve the current merge request.') } types MergeRequest condition do - quick_action_target.persisted? && quick_action_target.can_be_approved_by?(current_user) + quick_action_target.persisted? && quick_action_target.eligible_for_approval_by?(current_user) end command :approve do success = ::MergeRequests::ApprovalService.new(project: quick_action_target.project, current_user: current_user).execute(quick_action_target) @@ -190,7 +190,7 @@ module Gitlab explanation { _('Unapprove the current merge request.') } types MergeRequest condition do - quick_action_target.persisted? && quick_action_target.can_be_unapproved_by?(current_user) + quick_action_target.persisted? && quick_action_target.eligible_for_unapproval_by?(current_user) end command :unapprove do success = ::MergeRequests::RemoveApprovalService.new(project: quick_action_target.project, current_user: current_user).execute(quick_action_target) diff --git a/lib/gitlab/redis/duplicate_jobs.rb b/lib/gitlab/redis/duplicate_jobs.rb index beb3ba1abee..c76d298da18 100644 --- a/lib/gitlab/redis/duplicate_jobs.rb +++ b/lib/gitlab/redis/duplicate_jobs.rb @@ -18,9 +18,11 @@ module Gitlab # `Sidekiq.redis` is a namespaced redis connection. This means keys are actually being stored under # "resque:gitlab:resque:gitlab:duplicate:". For backwards compatibility, we make the secondary store # namespaced in the same way, but omit it from the primary so keys have proper format there. + # rubocop:disable Cop/RedisQueueUsage secondary_store = ::Redis::Namespace.new( Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE, redis: ::Redis.new(Gitlab::Redis::Queues.params) ) + # rubocop:enable Cop/RedisQueueUsage MultiStore.new(primary_store, secondary_store, name.demodulize) end diff --git a/lib/gitlab/redis/sidekiq_status.rb b/lib/gitlab/redis/sidekiq_status.rb index d4362c7cad8..9b8bbf5a0ad 100644 --- a/lib/gitlab/redis/sidekiq_status.rb +++ b/lib/gitlab/redis/sidekiq_status.rb @@ -14,7 +14,7 @@ module Gitlab def redis primary_store = ::Redis.new(Gitlab::Redis::SharedState.params) - secondary_store = ::Redis.new(Gitlab::Redis::Queues.params) + secondary_store = ::Redis.new(Gitlab::Redis::Queues.params) # rubocop:disable Cop/RedisQueueUsage MultiStore.new(primary_store, secondary_store, name.demodulize) end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 10c03103899..4f76cce2c7d 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -414,8 +414,10 @@ module Gitlab # Based on Jira's project key format # https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html + # Avoids linking CVE IDs (https://cve.mitre.org/cve/identifiers/syntaxchange.html#new) as Jira issues. + # CVE IDs use the format of CVE-YYYY-NNNNNNN def jira_issue_key_regex - @jira_issue_key_regex ||= /[A-Z][A-Z_0-9]+-\d+/ + @jira_issue_key_regex ||= /(?!CVE-\d+-\d+)[A-Z][A-Z_0-9]+-\d+/ end def jira_issue_key_project_key_extraction_regex diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index 33c7d96c45b..baf48fd0dc1 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -41,7 +41,7 @@ module Gitlab smembers, exists = with do |redis| redis.multi do |multi| multi.smembers(full_key) - multi.exists(full_key) + multi.exists?(full_key) # rubocop:disable CodeReuse/ActiveRecord end end @@ -58,7 +58,7 @@ module Gitlab full_key = cache_key(key) with do |redis| - exists = redis.exists(full_key) + exists = redis.exists?(full_key) # rubocop:disable CodeReuse/ActiveRecord write(key, yield) unless exists redis.sscan_each(full_key, match: pattern) diff --git a/lib/gitlab/request_endpoints.rb b/lib/gitlab/request_endpoints.rb index 157c0f91e65..4efafaa0ac2 100644 --- a/lib/gitlab/request_endpoints.rb +++ b/lib/gitlab/request_endpoints.rb @@ -8,6 +8,7 @@ module Gitlab # but if they weren't, the routes will be drawn and available for the rest of # application. API::API.compile! + API::API.reset_routes! API::API.routes.select { |route| route.app.options[:for] < API::Base } end diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index 5b1341207fd..6d95cb9a87b 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -25,9 +25,9 @@ module Gitlab if matches.one? matches.first elsif matches.none? - raise UnknownProcessError, "Failed to identify runtime for process #{Process.pid} (#{$0})" + raise UnknownProcessError, "Failed to identify runtime for process #{Process.pid} (#{$PROGRAM_NAME})" else - raise AmbiguousProcessError, "Ambiguous runtime #{matches} for process #{Process.pid} (#{$0})" + raise AmbiguousProcessError, "Ambiguous runtime #{matches} for process #{Process.pid} (#{$PROGRAM_NAME})" end end diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb index 4c5fae87420..360bbc073c5 100644 --- a/lib/gitlab/search/query.rb +++ b/lib/gitlab/search/query.rb @@ -40,19 +40,24 @@ module Gitlab query_tokens = parse_raw_query filters = @filters.each_with_object([]) do |filter, parsed_filters| - match = query_tokens.find { |part| part =~ /\A-?#{filter[:name]}:/ } + matches = query_tokens.select { |part| part =~ /\A-?#{filter[:name]}:/ } - next unless match + next unless matches.any? - input = match.split(':')[1..].join - next if input.empty? + matches.each do |match| + query_filter = filter.dup - filter[:negated] = match.start_with?("-") - filter[:value] = parse_filter(filter, input.gsub(QUOTES_REGEXP, '')) - filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?') - fragments << match + input = match.split(':')[1..].join - parsed_filters << filter + next if input.empty? + + query_filter[:negated] = match.start_with?("-") + query_filter[:value] = parse_filter(query_filter, input.gsub(QUOTES_REGEXP, '')) + query_filter[:regex_value] = Regexp.escape(query_filter[:value]).gsub('\*', '.*?') + + fragments << match + parsed_filters << query_filter + end end query = (query_tokens - fragments).join(' ') diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 23c23393bc8..c7818cb3418 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -28,7 +28,7 @@ module Gitlab end def exist?(key) - with { |redis| redis.exists(cache_key(key)) } + with { |redis| redis.exists?(cache_key(key)) } # rubocop:disable CodeReuse/ActiveRecord end def write(key, value) @@ -59,7 +59,7 @@ module Gitlab with do |redis| redis.multi do |multi| multi.sismember(full_key, value) - multi.exists(full_key) + multi.exists?(full_key) # rubocop:disable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index ac9a7d25fc2..3e7bdfbe89a 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -53,8 +53,33 @@ module Gitlab end end + def cron_jobs + @cron_jobs ||= begin + Gitlab.config.load_dynamic_cron_schedules! + + # Load recurring jobs from gitlab.yml + # UGLY Hack to get nested hash from settingslogic + jobs = Gitlab::Json.parse(Gitlab.config.cron_jobs.to_json) + + jobs.delete('poll_interval') # Would be interpreted as a job otherwise + + # UGLY hack: Settingslogic doesn't allow 'class' key + required_keys = %w[job_class cron] + jobs.each do |k, v| + if jobs[k] && required_keys.all? { |s| jobs[k].key?(s) } + jobs[k]['class'] = jobs[k].delete('job_class') + else + jobs.delete(k) + Gitlab::AppLogger.error("Invalid cron_jobs config key: '#{k}'. Check your gitlab config file.") + end + end + + jobs + end + end + def cron_workers - @cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize } + @cron_workers ||= cron_jobs.map { |job_name, options| options['class'].constantize } end def workers diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index 24e2eca420e..b8f86b92844 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -51,9 +51,10 @@ module Gitlab def refresh_state(phase) @phase = PHASE.fetch(phase) - @current_rss = get_rss - @soft_limit_rss = get_soft_limit_rss - @hard_limit_rss = get_hard_limit_rss + @current_rss = get_rss_kb + @soft_limit_rss = get_soft_limit_rss_kb + @hard_limit_rss = get_hard_limit_rss_kb + @memory_total = get_memory_total_kb # track the current state as prometheus gauges @metrics[:sidekiq_memory_killer_phase].set({}, @phase) @@ -107,6 +108,8 @@ module Gitlab end def restart_sidekiq + return if Feature.enabled?(:sidekiq_memory_killer_read_only_mode, type: :ops) + # Tell Sidekiq to stop fetching new jobs # We first SIGNAL and then wait given time # We also monitor a number of running jobs and allow to restart early @@ -176,6 +179,7 @@ module Gitlab current_rss: @current_rss, soft_limit_rss: @soft_limit_rss, hard_limit_rss: @hard_limit_rss, + memory_total_kb: @memory_total, reason: reason, running_jobs: running_jobs) @@ -212,18 +216,19 @@ module Gitlab end end - def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) - return 0 unless status&.zero? + def get_memory_total_kb + Gitlab::Metrics::System.memory_total / 1.kilobytes + end - output.to_i + def get_rss_kb + Gitlab::Metrics::System.memory_usage_rss / 1.kilobytes end - def get_soft_limit_rss + def get_soft_limit_rss_kb SOFT_LIMIT_RSS_KB + rss_increase_by_jobs end - def get_hard_limit_rss + def get_hard_limit_rss_kb HARD_LIMIT_RSS_KB end diff --git a/lib/gitlab/sidekiq_daemon/monitor.rb b/lib/gitlab/sidekiq_daemon/monitor.rb index 1f1d63877b5..655e95c82d3 100644 --- a/lib/gitlab/sidekiq_daemon/monitor.rb +++ b/lib/gitlab/sidekiq_daemon/monitor.rb @@ -182,7 +182,7 @@ module Gitlab def cancelled?(jid) ::Gitlab::Redis::SharedState.with do |redis| - redis.exists(self.class.cancel_job_key(jid)) + redis.exists?(self.class.cancel_job_key(jid)) # rubocop:disable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index fd3a5f715e8..b20f639ce85 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -7,7 +7,7 @@ module Gitlab # The result of this method should be passed to # Sidekiq's `config.server_middleware` method # eg: `config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator)` - def self.server_configurator(metrics: true, arguments_logger: true, memory_killer: true) + def self.server_configurator(metrics: true, arguments_logger: true) lambda do |chain| # Size limiter should be placed at the top chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Server @@ -27,7 +27,6 @@ module Gitlab end chain.add ::Gitlab::SidekiqMiddleware::ArgumentsLogger if arguments_logger - chain.add ::Gitlab::SidekiqMiddleware::MemoryKiller if memory_killer chain.add ::Gitlab::SidekiqMiddleware::RequestStoreMiddleware chain.add ::Gitlab::SidekiqMiddleware::ExtraDoneLogMetadata chain.add ::Gitlab::SidekiqMiddleware::BatchLoader diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index ab126ea4749..d42bd672bac 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -282,7 +282,7 @@ module Gitlab Gitlab::Redis::DuplicateJobs.with { |redis| yield redis } else # Keep the old behavior intact if neither feature flag is turned on - Sidekiq.redis { |redis| yield redis } + Sidekiq.redis { |redis| yield redis } # rubocop:disable Cop/SidekiqRedisCall end end end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb deleted file mode 100644 index 0b38c98f710..00000000000 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ /dev/null @@ -1,91 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module SidekiqMiddleware - class MemoryKiller - # Default the RSS limit to 0, meaning the MemoryKiller is disabled - MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i - # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit - GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i - # Wait 30 seconds for running jobs to finish during graceful shutdown - SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i - - # Create a mutex used to ensure there will be only one thread waiting to - # shut Sidekiq down - MUTEX = Mutex.new - - attr_reader :worker - - def call(worker, job, queue) - yield - - @worker = worker - current_rss = get_rss - - return unless MAX_RSS > 0 && current_rss > MAX_RSS - - Thread.new do - # Return if another thread is already waiting to shut Sidekiq down - next unless MUTEX.try_lock - - warn("Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ - " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}") - - warn("Sidekiq worker PID-#{pid} will stop fetching new jobs"\ - " in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later") - - # Wait `GRACE_TIME` to give the memory intensive job time to finish. - # Then, tell Sidekiq to stop fetching new jobs. - wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs') - - # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. - # Then, tell Sidekiq to gracefully shut down by giving jobs a few more - # moments to finish, killing and requeuing them if they didn't, and - # then terminating itself. Sidekiq will replicate the TERM to all its - # children if it can. - wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') - - # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. - # Kill the whole pgroup, so we can be sure no children are left behind - wait_and_signal_pgroup(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') - end - end - - private - - def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) - return 0 unless status == 0 - - output.to_i - end - - # If this sidekiq process is pgroup leader, signal to the whole pgroup - def wait_and_signal_pgroup(time, signal, explanation) - return wait_and_signal(time, signal, explanation) unless Process.getpgrp == pid - - warn("waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) - sleep(time) - - warn("sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) - Process.kill(signal, 0) - end - - def wait_and_signal(time, signal, explanation) - warn("waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) - sleep(time) - - warn("sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) - Process.kill(signal, pid) - end - - def pid - Process.pid - end - - def warn(message, signal: nil) - Sidekiq.logger.warn(class: worker.class.name, pid: pid, signal: signal, message: message) - end - end - end -end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 9d08d236720..17234bdf519 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -126,7 +126,7 @@ module Gitlab Gitlab::Redis::SidekiqStatus.with { |redis| yield redis } else # Keep the old behavior intact if neither feature flag is turned on - Sidekiq.redis { |redis| yield redis } + Sidekiq.redis { |redis| yield redis } # rubocop:disable Cop/SidekiqRedisCall end end private_class_method :with_redis diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index ef368767689..1527fd263e0 100644 --- a/lib/gitlab/slash_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -21,12 +21,16 @@ module Gitlab title = match[:title] description = match[:description].to_s.rstrip - issue = create_issue(title: title, description: description) + result = create_issue(title: title, description: description) - if issue.persisted? - presenter(issue).present + if result.success? + presenter(result[:issue]).present + elsif result[:issue] + presenter(result[:issue]).display_errors else - presenter(issue).display_errors + Gitlab::SlashCommands::Presenters::Error.new( + result.errors.join(', ') + ).message end end diff --git a/lib/gitlab/tracking/service_ping_context.rb b/lib/gitlab/tracking/service_ping_context.rb new file mode 100644 index 00000000000..393cd647e7f --- /dev/null +++ b/lib/gitlab/tracking/service_ping_context.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + class ServicePingContext + SCHEMA_URL = 'iglu:com.gitlab/gitlab_service_ping/jsonschema/1-0-0' + ALLOWED_SOURCES = %i[redis_hll].freeze + + def initialize(data_source:, event:) + unless ALLOWED_SOURCES.include?(data_source) + raise ArgumentError, "#{data_source} is not acceptable data source for ServicePingContext" + end + + @payload = { + data_source: data_source, + event_name: event + } + end + + def to_context + SnowplowTracker::SelfDescribingJson.new(SCHEMA_URL, @payload) + end + end + end +end diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb index cf48aa49938..30f2efc8638 100644 --- a/lib/gitlab/usage/metric.rb +++ b/lib/gitlab/usage/metric.rb @@ -46,10 +46,7 @@ module Gitlab end def instrumentation_object - @instrumentation_object ||= instrumentation_class.constantize.new( - time_frame: definition.time_frame, - options: definition.attributes[:options] - ) + @instrumentation_object ||= instrumentation_class.constantize.new(definition.attributes) end end end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 2c50678c6bf..d6b1e62c84f 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -5,7 +5,7 @@ module Gitlab class MetricDefinition METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze - AVAILABLE_STATUSES = %w[active data_available implemented deprecated].to_set.freeze + AVAILABLE_STATUSES = %w[active data_available implemented deprecated broken].to_set.freeze VALID_SERVICE_PING_STATUSES = %w[active data_available implemented deprecated broken].to_set.freeze InvalidError = Class.new(RuntimeError) diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 11e2fd22638..cd72f16d46d 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -13,62 +13,72 @@ module Gitlab end def all_time_data - aggregated_metrics_data(start_date: nil, end_date: nil, time_frame: Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME) + aggregated_metrics_data(Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME) end def monthly_data - aggregated_metrics_data(**monthly_time_range.merge(time_frame: Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME)) + aggregated_metrics_data(Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME) end def weekly_data - aggregated_metrics_data(**weekly_time_range.merge(time_frame: Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME)) + aggregated_metrics_data(Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME) + end + + def calculate_count_for_aggregation(aggregation:, time_frame:) + with_validate_configuration(aggregation, time_frame) do + source = SOURCES[aggregation[:source]] + + if aggregation[:operator] == UNION_OF_AGGREGATED_METRICS + source.calculate_metrics_union(**time_constraints(time_frame).merge(metric_names: aggregation[:events], recorded_at: recorded_at)) + else + source.calculate_metrics_intersections(**time_constraints(time_frame).merge(metric_names: aggregation[:events], recorded_at: recorded_at)) + end + end + rescue Gitlab::UsageDataCounters::HLLRedisCounter::EventError, AggregatedMetricError => error + failure(error) end private attr_accessor :aggregated_metrics, :recorded_at - def aggregated_metrics_data(start_date:, end_date:, time_frame:) + def aggregated_metrics_data(time_frame) aggregated_metrics.each_with_object({}) do |aggregation, data| next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], type: :development) next unless aggregation[:time_frame].include?(time_frame) - case aggregation[:source] - when REDIS_SOURCE - if time_frame == Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME - data[aggregation[:name]] = Gitlab::Utils::UsageData::FALLBACK - Gitlab::ErrorTracking - .track_and_raise_for_dev_exception( - DisallowedAggregationTimeFrame.new("Aggregation time frame: 'all' is not allowed for aggregation with source: '#{REDIS_SOURCE}'") - ) - else - data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) - end - when DATABASE_SOURCE - data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) - else - Gitlab::ErrorTracking - .track_and_raise_for_dev_exception(UnknownAggregationSource.new("Aggregation source: '#{aggregation[:source]}' must be included in #{SOURCES.keys}")) - - data[aggregation[:name]] = Gitlab::Utils::UsageData::FALLBACK - end + data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, time_frame: time_frame) end end - def calculate_count_for_aggregation(aggregation:, start_date:, end_date:) - source = SOURCES[aggregation[:source]] - - case aggregation[:operator] - when UNION_OF_AGGREGATED_METRICS - source.calculate_metrics_union(metric_names: aggregation[:events], start_date: start_date, end_date: end_date, recorded_at: recorded_at) - when INTERSECTION_OF_AGGREGATED_METRICS - source.calculate_metrics_intersections(metric_names: aggregation[:events], start_date: start_date, end_date: end_date, recorded_at: recorded_at) - else - Gitlab::ErrorTracking - .track_and_raise_for_dev_exception(UnknownAggregationOperator.new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}")) - Gitlab::Utils::UsageData::FALLBACK + def with_validate_configuration(aggregation, time_frame) + source = aggregation[:source] + + unless ALLOWED_METRICS_AGGREGATIONS.include?(aggregation[:operator]) + return failure( + UnknownAggregationOperator + .new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}") + ) end - rescue Gitlab::UsageDataCounters::HLLRedisCounter::EventError, AggregatedMetricError => error + + unless SOURCES[source] + return failure( + UnknownAggregationSource + .new("Aggregation source: '#{source}' must be included in #{SOURCES.keys}") + ) + end + + if time_frame == Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME && source == REDIS_SOURCE + return failure( + DisallowedAggregationTimeFrame + .new("Aggregation time frame: 'all' is not allowed for aggregation with source: '#{REDIS_SOURCE}'") + ) + end + + yield + end + + def failure(error) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) Gitlab::Utils::UsageData::FALLBACK end @@ -82,6 +92,17 @@ module Gitlab def load_yaml_from_path(path) YAML.safe_load(File.read(path), aliases: true)&.map(&:with_indifferent_access) end + + def time_constraints(time_frame) + case time_frame + when Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME + monthly_time_range + when Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME + weekly_time_range + when Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME + { start_date: nil, end_date: nil } + end + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb b/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb new file mode 100644 index 00000000000..63ead5a8cb0 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + # Usage example + # + # In metric YAML definition: + # + # instrumentation_class: AggregatedMetric + # data_source: redis_hll + # options: + # aggregate: + # operator: OR + # attribute: user_id + # events: + # - 'incident_management_alert_status_changed' + # - 'incident_management_alert_assigned' + # - 'incident_management_alert_todo' + # - 'incident_management_alert_create_incident' + + class AggregatedMetric < BaseMetric + FALLBACK = -1 + + def initialize(metric_definition) + super + @source = parse_data_source_to_legacy_value(metric_definition) + @aggregate = options.fetch(:aggregate, {}) + end + + def value + alt_usage_data(fallback: FALLBACK) do + Aggregates::Aggregate + .new(Time.current) + .calculate_count_for_aggregation( + aggregation: aggregate_config, + time_frame: time_frame + ) + end + end + + def suggested_name + Gitlab::Usage::Metrics::NameSuggestion.for(:alt) + end + + private + + attr_accessor :source, :aggregate + + # TODO: This method is a temporary measure that + # handles backwards compatibility until + # point 5 from is resolved https://gitlab.com/gitlab-org/gitlab/-/issues/370963#implementation + def parse_data_source_to_legacy_value(metric_definition) + return 'redis' if metric_definition[:data_source] == 'redis_hll' + + metric_definition[:data_source] + end + + def aggregate_config + { + source: source, + events: options[:events], + operator: aggregate[:operator] + } + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb index 5e20766b1b4..55da2315e45 100644 --- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb @@ -23,9 +23,9 @@ module Gitlab attr_reader :metric_available end - def initialize(time_frame:, options: {}) - @time_frame = time_frame - @options = options + def initialize(metric_definition) + @time_frame = metric_definition.fetch(:time_frame) + @options = metric_definition.fetch(:options, {}) end def instrumentation diff --git a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb index 67dc1455b23..642b67a3b02 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb @@ -7,7 +7,7 @@ module Gitlab class CountBulkImportsEntitiesMetric < DatabaseMetric operation :count - def initialize(time_frame:, options: {}) + def initialize(metric_definition) super if source_type.present? && !source_type.in?(allowed_source_types) diff --git a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb index c5498ce530f..d485e8b4f72 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb @@ -7,7 +7,7 @@ module Gitlab class CountImportedProjectsMetric < DatabaseMetric operation :count - def initialize(time_frame:, options: {}) + def initialize(metric_definition) super raise ArgumentError, "import_type options attribute is required" unless import_type.present? diff --git a/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_disabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_disabled_metric.rb new file mode 100644 index 00000000000..0c421dc3311 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_disabled_metric.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class DistinctCountProjectsWithExpirationPolicyDisabledMetric < DatabaseMetric + operation :distinct_count, column: :project_id + + start { Project.minimum(:id) } + finish { Project.maximum(:id) } + + cache_start_and_finish_as :project_id + + relation { ::ContainerExpirationPolicy.where(enabled: false) } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb index 0f4b903b99c..7c646281598 100644 --- a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb @@ -28,9 +28,8 @@ module Gitlab end end - def initialize(time_frame: 'none', options: {}) - @time_frame = time_frame - @options = options + def initialize(metric_definition) + super(metric_definition.reverse_merge(time_frame: 'none')) end def value diff --git a/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_direct_installations_count_metric.rb b/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_direct_installations_count_metric.rb new file mode 100644 index 00000000000..f22ee2aa3f5 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_direct_installations_count_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GitlabForJiraAppDirectInstallationsCountMetric < DatabaseMetric + operation :count + + relation { JiraConnectInstallation.direct_installations } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_proxy_installations_count_metric.rb b/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_proxy_installations_count_metric.rb new file mode 100644 index 00000000000..222a69faf8b --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_proxy_installations_count_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GitlabForJiraAppProxyInstallationsCountMetric < DatabaseMetric + operation :count + + relation { JiraConnectInstallation.proxy_installations } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/merge_request_widget_extension_metric.rb b/lib/gitlab/usage/metrics/instrumentations/merge_request_widget_extension_metric.rb new file mode 100644 index 00000000000..e2fdb3462c5 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/merge_request_widget_extension_metric.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + # Usage example + # + # In metric YAML definition: + # + # instrumentation_class: MergeRequestWidgetExtensionMetric + # options: + # event: expand + # widget: terraform + # + class MergeRequestWidgetExtensionMetric < RedisMetric + extend ::Gitlab::Utils::Override + + def validate_options! + raise ArgumentError, "'event' option is required" unless metric_event.present? + raise ArgumentError, "'widget' option is required" unless widget_name.present? + end + + def widget_name + options[:widget] + end + + override :prefix + def prefix + 'i_code_review_merge_request_widget' + end + + private + + override :redis_key + def redis_key + "#{USAGE_PREFIX}#{prefix}_#{widget_name}_count_#{metric_event}".upcase + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb index bb27cca1bb9..17009f7638e 100644 --- a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb @@ -12,7 +12,7 @@ module Gitlab # events: # - g_analytics_valuestream # end - def initialize(time_frame:, options: {}) + def initialize(metric_definition) super raise ArgumentError, "options events are required" unless metric_events.present? diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb index 26d963e2407..ae3326fa845 100644 --- a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb @@ -17,12 +17,17 @@ module Gitlab include Gitlab::UsageDataCounters::RedisCounter USAGE_PREFIX = "USAGE_" + OPTIONS_PREFIX_KEY = :prefix - def initialize(time_frame:, options: {}) + def initialize(metric_definition) super + validate_options! + end + + def validate_options! raise ArgumentError, "'event' option is required" unless metric_event.present? - raise ArgumentError, "'prefix' option is required" unless prefix.present? + raise ArgumentError, "'prefix' option is required" unless options.has_key?(OPTIONS_PREFIX_KEY) end def metric_event @@ -30,7 +35,7 @@ module Gitlab end def prefix - options[:prefix] + options[OPTIONS_PREFIX_KEY] end def include_usage_prefix? @@ -50,9 +55,10 @@ module Gitlab private def redis_key - key = "#{prefix}_#{metric_event}".upcase + key = metric_event.dup + key.prepend("#{prefix}_") if prefix key.prepend(USAGE_PREFIX) if include_usage_prefix? - key + key.upcase end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/work_items_activity_aggregated_metric.rb b/lib/gitlab/usage/metrics/instrumentations/work_items_activity_aggregated_metric.rb new file mode 100644 index 00000000000..d045265495a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/work_items_activity_aggregated_metric.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class WorkItemsActivityAggregatedMetric < AggregatedMetric + available? { Feature.enabled?(:track_work_items_activity) } + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index e2232dc5e2a..87ccb9a31da 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -21,18 +21,18 @@ module Gitlab MAX_GENERATION_TIME_FOR_SAAS = 40.hours CE_MEMOIZED_VALUES = %i( - issue_minimum_id - issue_maximum_id - project_minimum_id - project_maximum_id - user_minimum_id - user_maximum_id - deployment_minimum_id - deployment_maximum_id - auth_providers - aggregated_metrics - recorded_at - ).freeze + issue_minimum_id + issue_maximum_id + project_minimum_id + project_maximum_id + user_minimum_id + user_maximum_id + deployment_minimum_id + deployment_maximum_id + auth_providers + aggregated_metrics + recorded_at + ).freeze class << self include Gitlab::Utils::UsageData @@ -346,7 +346,6 @@ module Gitlab start = minimum_id(Project) finish = maximum_id(Project) - results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish) # rubocop: disable UsageData/LargeTable base = ::ContainerExpirationPolicy.active # rubocop: enable UsageData/LargeTable diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb index eae1c593a8f..37c6e1af7c0 100644 --- a/lib/gitlab/usage_data_counters.rb +++ b/lib/gitlab/usage_data_counters.rb @@ -2,25 +2,24 @@ module Gitlab module UsageDataCounters - COUNTERS = [ - WikiPageCounter, - NoteCounter, - SnippetCounter, - SearchCounter, - CycleAnalyticsCounter, - ProductivityAnalyticsCounter, - SourceCodeCounter, - KubernetesAgentCounter, - MergeRequestWidgetExtensionCounter - ].freeze + COUNTERS = [].freeze COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES = [ PackageEventCounter, MergeRequestCounter, DesignsCounter, DiffsCounter, + KubernetesAgentCounter, + NoteCounter, + SearchCounter, ServiceUsageDataCounter, - WebIdeCounter + WebIdeCounter, + WikiPageCounter, + SnippetCounter, + CycleAnalyticsCounter, + ProductivityAnalyticsCounter, + SourceCodeCounter, + MergeRequestWidgetExtensionCounter ].freeze UsageDataCounterError = Class.new(StandardError) @@ -28,13 +27,11 @@ module Gitlab class << self def unmigrated_counters - # we are using the #counters method instead of the COUNTERS const - # to make sure it's working correctly for `ee` version of UsageDataCounters - counters - self::COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES + self::COUNTERS end def counters - self::COUNTERS + self::COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES + unmigrated_counters + migrated_counters end def count(event_name) @@ -46,6 +43,12 @@ module Gitlab raise UnknownEvent, "Cannot find counter for event #{event_name}" end + + private + + def migrated_counters + COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES + end end end end diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index 61c071c8738..1e8918c7c96 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -34,6 +34,21 @@ module Gitlab::UsageDataCounters Gitlab::Template::GitlabCiYmlTemplate.find(template_name.chomp('.gitlab-ci.yml'))&.full_name end + def all_included_templates(template_name) + expanded_template_name = expand_template_name(template_name) + results = [expanded_template_name].tap do |result| + template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name.chomp('.gitlab-ci.yml')) + data = YAML.safe_load(template.content, aliases: true) + [data['include']].compact.flatten.each do |ci_include| + if ci_include_template = ci_include['template'] + result.concat(all_included_templates(ci_include_template)) + end + end + end + + results.uniq.sort_by { _1['name'] } + end + private def template_to_event_name(template) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index f0cb9bcbe94..24a87ae01f4 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -19,18 +19,14 @@ module Gitlab ALLOWED_AGGREGATIONS = %i(daily weekly).freeze CATEGORIES_FOR_TOTALS = %w[ - analytics compliance - epics_usage error_tracking ide_edit - incident_management - issues_edit pipeline_authoring - quickactions ].freeze CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS = %w[ + analytics ci_users deploy_token_packages code_review @@ -38,9 +34,13 @@ module Gitlab error_tracking ide_edit importer + incident_management incident_management_alerts + issues_edit kubernetes_agent + manage pipeline_authoring + quickactions search secure snippets diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml index 76c97a974d7..85524c766ca 100644 --- a/lib/gitlab/usage_data_counters/known_events/analytics.yml +++ b/lib/gitlab/usage_data_counters/known_events/analytics.yml @@ -10,54 +10,18 @@ category: analytics redis_slot: analytics aggregation: weekly -- name: p_analytics_merge_request - category: analytics - redis_slot: analytics - aggregation: weekly - name: i_analytics_instance_statistics category: analytics redis_slot: analytics aggregation: weekly -- name: g_analytics_contribution - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_insights - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_issues - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_productivity - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_valuestream - category: analytics - redis_slot: analytics - aggregation: weekly - name: p_analytics_pipelines category: analytics redis_slot: analytics aggregation: weekly -- name: p_analytics_code_reviews - category: analytics - redis_slot: analytics - aggregation: weekly - name: p_analytics_valuestream category: analytics redis_slot: analytics aggregation: weekly -- name: p_analytics_insights - category: analytics - redis_slot: analytics - aggregation: weekly -- name: p_analytics_issues - category: analytics - redis_slot: analytics - aggregation: weekly - name: p_analytics_repo category: analytics redis_slot: analytics @@ -86,23 +50,3 @@ category: analytics redis_slot: analytics aggregation: weekly -- name: g_analytics_ci_cd_release_statistics - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_ci_cd_deployment_frequency - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_ci_cd_lead_time - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_ci_cd_time_to_restore_service - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_ci_cd_change_failure_rate - category: analytics - redis_slot: analytics - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index 10e36a75a3a..5b80f6c6c0d 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -99,6 +99,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_security_coverage_fuzzing_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_security_dast_on_demand_api_scan category: ci_templates redis_slot: ci_templates @@ -499,27 +503,11 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_jobs_dast_default_branch_deploy - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_load_performance_testing - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_helm_2to3 - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_sast - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_secret_detection +- name: p_ci_templates_implicit_jobs_browser_performance_testing category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_jobs_license_scanning +- name: p_ci_templates_implicit_jobs_build category: ci_templates redis_slot: ci_templates aggregation: weekly @@ -531,47 +519,7 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_jobs_deploy_ecs - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_deploy_ec2 - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_license_scanning_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_deploy - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_build - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_browser_performance_testing - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_dependency_scanning_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_test - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_sast_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_sast_iac - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_secret_detection_latest +- name: p_ci_templates_implicit_jobs_dast_default_branch_deploy category: ci_templates redis_slot: ci_templates aggregation: weekly @@ -579,63 +527,35 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_jobs_deploy_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_browser_performance_testing_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_cf_provision - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_build_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_sast_iac_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_security_sast - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_security_dast_runner_validation - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_security_dast_on_demand_scan +- name: p_ci_templates_implicit_jobs_deploy category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_secret_detection +- name: p_ci_templates_implicit_jobs_deploy_ec2 category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_license_scanning +- name: p_ci_templates_implicit_jobs_deploy_ecs category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_dast_on_demand_api_scan +- name: p_ci_templates_implicit_jobs_helm_2to3 category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_coverage_fuzzing +- name: p_ci_templates_implicit_jobs_license_scanning category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_api_fuzzing_latest +- name: p_ci_templates_implicit_jobs_sast category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_secure_binaries +- name: p_ci_templates_implicit_jobs_secret_detection category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_dast_api +- name: p_ci_templates_implicit_jobs_test category: ci_templates redis_slot: ci_templates aggregation: weekly @@ -643,11 +563,7 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_dast_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_security_sast_iac +- name: p_ci_templates_implicit_security_dast category: ci_templates redis_slot: ci_templates aggregation: weekly @@ -655,27 +571,15 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_dast_api_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_security_container_scanning_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_security_api_fuzzing - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_security_dast +- name: p_ci_templates_implicit_security_license_scanning category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_fortify_fod_sast +- name: p_ci_templates_implicit_security_sast category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_sast_iac_latest +- name: p_ci_templates_implicit_security_secret_detection category: ci_templates redis_slot: ci_templates aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 29b231f88f8..c13c7657576 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -84,11 +84,6 @@ redis_slot: incident_management category: incident_management aggregation: weekly -- name: incident_management_incident_published - redis_slot: incident_management - category: incident_management - aggregation: weekly - feature_flag: usage_data_incident_management_incident_published - name: incident_management_incident_relate redis_slot: incident_management category: incident_management @@ -114,29 +109,11 @@ redis_slot: incident_management category: incident_management aggregation: weekly -# Incident management linked resources -- name: incident_management_issuable_resource_link_created - redis_slot: incident_management - category: incident_management - aggregation: weekly -- name: incident_management_issuable_resource_link_deleted - redis_slot: incident_management - category: incident_management - aggregation: weekly -- name: incident_management_issuable_resource_link_visited - redis_slot: incident_management - category: incident_management - aggregation: weekly # Incident management alerts - name: incident_management_alert_create_incident redis_slot: incident_management category: incident_management_alerts aggregation: weekly -# Incident management on-call -- name: i_incident_management_oncall_notification_sent - redis_slot: incident_management - category: incident_management_oncall - aggregation: weekly # Testing category - name: i_testing_test_case_parsed category: testing @@ -150,7 +127,6 @@ category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_ci_i_testing_test_report_uploaded # Project Management group - name: g_project_management_issue_title_changed category: issues_edit @@ -192,14 +168,6 @@ category: issues_edit redis_slot: project_management aggregation: daily -- name: g_project_management_issue_iteration_changed - category: issues_edit - redis_slot: project_management - aggregation: daily -- name: g_project_management_issue_weight_changed - category: issues_edit - redis_slot: project_management - aggregation: daily - name: g_project_management_issue_cross_referenced category: issues_edit redis_slot: project_management @@ -228,18 +196,6 @@ category: issues_edit redis_slot: project_management aggregation: daily -- name: g_project_management_issue_added_to_epic - category: issues_edit - redis_slot: project_management - aggregation: daily -- name: g_project_management_issue_removed_from_epic - category: issues_edit - redis_slot: project_management - aggregation: daily -- name: g_project_management_issue_changed_epic - category: issues_edit - redis_slot: project_management - aggregation: daily - name: g_project_management_issue_designs_added category: issues_edit redis_slot: project_management @@ -276,20 +232,11 @@ category: issues_edit redis_slot: project_management aggregation: daily -- name: g_project_management_issue_health_status_changed - category: issues_edit - redis_slot: project_management - aggregation: daily - name: g_project_management_issue_cloned category: issues_edit redis_slot: project_management aggregation: daily # Secrets Management -- name: i_ci_secrets_management_vault_build_created - category: ci_secrets_management - redis_slot: ci_secrets_management - aggregation: weekly - feature_flag: usage_data_i_ci_secrets_management_vault_build_created - name: i_snippets_show category: snippets redis_slot: snippets @@ -342,11 +289,6 @@ category: geo redis_slot: geo aggregation: daily -# Growth -- name: users_clicking_registration_features_offer - category: growth - redis_slot: users - aggregation: weekly # Manage - name: unique_active_user category: manage diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml deleted file mode 100644 index dd6625a9cc9..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/epic_events.yml +++ /dev/null @@ -1,227 +0,0 @@ -# Epic events -# -# We are using the same slot of issue events 'project_management' for -# epic events to allow data aggregation. -# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405 -- name: g_project_management_epic_created - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -# content change events - -- name: project_management_users_unchecking_epic_task - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: project_management_users_checking_epic_task - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_updating_epic_titles - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_updating_epic_descriptions - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -# epic notes - -- name: g_project_management_users_creating_epic_notes - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_updating_epic_notes - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_destroying_epic_notes - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -# emoji - -- name: g_project_management_users_awarding_epic_emoji - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_removing_epic_emoji - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -# start date events - -- name: g_project_management_users_setting_epic_start_date_as_fixed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_updating_fixed_epic_start_date - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_setting_epic_start_date_as_inherited - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -# due date events - -- name: g_project_management_users_setting_epic_due_date_as_fixed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_updating_fixed_epic_due_date - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_setting_epic_due_date_as_inherited - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -# relationships - -- name: g_project_management_epic_issue_added - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_issue_removed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_issue_moved_from_project - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_updating_epic_parent - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_closed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_reopened - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: 'g_project_management_issue_promoted_to_epic' - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_setting_epic_confidential - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_setting_epic_visible - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_users_changing_labels - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_destroyed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_cross_referenced - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_epic_issue_added_from_epic - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_related_added - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_related_removed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_blocking_added - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_blocking_removed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_blocked_added - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_blocked_removed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index 58a0c0695af..69b348b9a22 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -1,17 +1,17 @@ --- -- name: i_quickactions_approve +- name: i_quickactions_assign_multiple category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_unapprove +- name: i_quickactions_approve category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_assign_single +- name: i_quickactions_unapprove category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_assign_multiple +- name: i_quickactions_assign_single category: quickactions redis_slot: quickactions aggregation: weekly @@ -31,18 +31,6 @@ category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_child_epic - category: quickactions - redis_slot: quickactions - aggregation: weekly -- name: i_quickactions_clear_weight - category: quickactions - redis_slot: quickactions - aggregation: weekly -- name: i_quickactions_clear_health_status - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_clone category: quickactions redis_slot: quickactions @@ -83,18 +71,10 @@ category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_epic - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_estimate category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_iteration - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_label category: quickactions redis_slot: quickactions @@ -115,14 +95,6 @@ category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_parent_epic - category: quickactions - redis_slot: quickactions - aggregation: weekly -- name: i_quickactions_promote - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_promote_to_incident category: quickactions redis_slot: quickactions @@ -131,14 +103,6 @@ category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_page - category: quickactions - redis_slot: quickactions - aggregation: weekly -- name: i_quickactions_publish - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_ready category: quickactions redis_slot: quickactions @@ -163,34 +127,18 @@ category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_remove_child_epic - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_remove_due_date category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_remove_epic - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_remove_estimate category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_remove_iteration - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_remove_milestone category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_remove_parent_epic - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_remove_time_spent category: quickactions redis_slot: quickactions @@ -275,19 +223,15 @@ category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_weight - category: quickactions - redis_slot: quickactions - aggregation: weekly -- name: i_quickactions_health_status +- name: i_quickactions_wip category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_wip +- name: i_quickactions_zoom category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_zoom +- name: i_quickactions_link category: quickactions redis_slot: quickactions aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/work_items.yml b/lib/gitlab/usage_data_counters/known_events/work_items.yml index 6cd7836ea94..ee828fc0f72 100644 --- a/lib/gitlab/usage_data_counters/known_events/work_items.yml +++ b/lib/gitlab/usage_data_counters/known_events/work_items.yml @@ -14,3 +14,16 @@ redis_slot: users aggregation: weekly feature_flag: track_work_items_activity +- name: users_updating_work_item_labels + category: work_items + redis_slot: users + aggregation: weekly + feature_flag: track_work_items_activity +- name: users_updating_work_item_iteration + # The event tracks an EE feature. + # It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics. + # It will report 0 for CE instances and should not be used with 'AND' aggregators. + category: work_items + redis_slot: users + aggregation: weekly + feature_flag: track_work_items_activity diff --git a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb index 99b4c082310..a0fd04596fc 100644 --- a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb @@ -6,6 +6,7 @@ module Gitlab WORK_ITEM_CREATED = 'users_creating_work_items' WORK_ITEM_TITLE_CHANGED = 'users_updating_work_item_title' WORK_ITEM_DATE_CHANGED = 'users_updating_work_item_dates' + WORK_ITEM_LABELS_CHANGED = 'users_updating_work_item_labels' class << self def track_work_item_created_action(author:) @@ -20,6 +21,10 @@ module Gitlab track_unique_action(WORK_ITEM_DATE_CHANGED, author) end + def track_work_item_labels_changed_action(author:) + track_unique_action(WORK_ITEM_LABELS_CHANGED, author) + end + private def track_unique_action(action, author) diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index c2983779603..4486ca53966 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -75,10 +75,6 @@ module Gitlab } end - def epics_deepest_relationship_level - { epics_deepest_relationship_level: 0 } - end - def topology_usage_data { duration_s: 0, diff --git a/lib/gitlab/utils/execution_tracker.rb b/lib/gitlab/utils/execution_tracker.rb index 6d48658853c..92398926e1b 100644 --- a/lib/gitlab/utils/execution_tracker.rb +++ b/lib/gitlab/utils/execution_tracker.rb @@ -3,7 +3,7 @@ module Gitlab module Utils class ExecutionTracker - MAX_RUNTIME = 30.seconds + MAX_RUNTIME = 60.seconds ExecutionTimeOutError = Class.new(StandardError) diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 19bdeefed7e..0b818b99ac7 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -291,14 +291,6 @@ module Gitlab end end - def epics_deepest_relationship_level - with_duration do - # rubocop: disable UsageData/LargeTable - { epics_deepest_relationship_level: ::Epic.deepest_relationship_level.to_i } - # rubocop: enable UsageData/LargeTable - end - end - private def prometheus_client(verify:) diff --git a/lib/gitlab/web_hooks.rb b/lib/gitlab/web_hooks.rb index 349c7a020cc..8c6de56292a 100644 --- a/lib/gitlab/web_hooks.rb +++ b/lib/gitlab/web_hooks.rb @@ -3,5 +3,6 @@ module Gitlab module WebHooks GITLAB_EVENT_HEADER = 'X-Gitlab-Event' + GITLAB_INSTANCE_HEADER = 'X-Gitlab-Instance' end end diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index 8acbfc144e9..f8a6980f208 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -37,7 +37,7 @@ module Gitlab !verified_signature || user.nil? - if user.verified_emails.include?(@email) && certificate_email == @email + if user.verified_emails.include?(@email.downcase) && certificate_email.casecmp?(@email) :verified else :unverified diff --git a/lib/prometheus/pid_provider.rb b/lib/prometheus/pid_provider.rb index d2563b4c806..05a2d3bd0c9 100644 --- a/lib/prometheus/pid_provider.rb +++ b/lib/prometheus/pid_provider.rb @@ -39,7 +39,7 @@ module Prometheus end def process_name - $0 + $PROGRAM_NAME end end end diff --git a/lib/quality/seeders/issues.rb b/lib/quality/seeders/issues.rb index 5d345dd30a1..bf42913e043 100644 --- a/lib/quality/seeders/issues.rb +++ b/lib/quality/seeders/issues.rb @@ -31,9 +31,9 @@ module Quality } params[:closed_at] = params[:created_at] + rand(35).days if params[:state] == 'closed' - issue = ::Issues::CreateService.new(project: project, current_user: team.sample, params: params, spam_params: nil).execute_without_rate_limiting + create_result = ::Issues::CreateService.new(project: project, current_user: team.sample, params: params, spam_params: nil).execute_without_rate_limiting - if issue.persisted? + if create_result.success? created_issues_count += 1 print '.' # rubocop:disable Rails/Output end diff --git a/lib/sidebars/groups/menus/packages_registries_menu.rb b/lib/sidebars/groups/menus/packages_registries_menu.rb index 61cd81711f8..873f11f8a5b 100644 --- a/lib/sidebars/groups/menus/packages_registries_menu.rb +++ b/lib/sidebars/groups/menus/packages_registries_menu.rb @@ -8,7 +8,7 @@ module Sidebars def configure_menu_items add_item(packages_registry_menu_item) add_item(container_registry_menu_item) - add_item(harbor_registry__menu_item) + add_item(harbor_registry_menu_item) add_item(dependency_proxy_menu_item) true end @@ -49,8 +49,10 @@ module Sidebars ) end - def harbor_registry__menu_item - if Feature.disabled?(:harbor_registry_integration) || context.group.harbor_integration.nil? + def harbor_registry_menu_item + if Feature.disabled?(:harbor_registry_integration) || + context.group.harbor_integration.nil? || + !context.group.harbor_integration.activated? return nil_menu_item(:harbor_registry) end diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index 63eea0ea500..2181d89262b 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -89,12 +89,12 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Google Cloud'), link: project_google_cloud_configuration_path(context.project), - active_routes: { controller: [ - :configuration, - :service_accounts, - :databases, - :deployments, - :gcp_regions + active_routes: { controller: %w[ + projects/google_cloud/configuration + projects/google_cloud/service_accounts + projects/google_cloud/databases + projects/google_cloud/deployments + projects/google_cloud/gcp_regions ] }, item_id: :google_cloud ) diff --git a/lib/sidebars/projects/menus/packages_registries_menu.rb b/lib/sidebars/projects/menus/packages_registries_menu.rb index 2ddffe42899..fc7c564574a 100644 --- a/lib/sidebars/projects/menus/packages_registries_menu.rb +++ b/lib/sidebars/projects/menus/packages_registries_menu.rb @@ -9,7 +9,7 @@ module Sidebars add_item(packages_registry_menu_item) add_item(container_registry_menu_item) add_item(infrastructure_registry_menu_item) - add_item(harbor_registry__menu_item) + add_item(harbor_registry_menu_item) true end @@ -65,8 +65,10 @@ module Sidebars ) end - def harbor_registry__menu_item - if Feature.disabled?(:harbor_registry_integration, context.project) || context.project.harbor_integration.nil? + def harbor_registry_menu_item + if Feature.disabled?(:harbor_registry_integration, context.project) || + context.project.harbor_integration.nil? || + !context.project.harbor_integration.activated? return ::Sidebars::NilMenuItem.new(item_id: :harbor_registry) end diff --git a/lib/sidebars/projects/menus/repository_menu.rb b/lib/sidebars/projects/menus/repository_menu.rb index 0a295f0f618..1b46323089c 100644 --- a/lib/sidebars/projects/menus/repository_menu.rb +++ b/lib/sidebars/projects/menus/repository_menu.rb @@ -85,6 +85,8 @@ module Sidebars end def contributors_menu_item + return false unless context.project.analytics_enabled? + ::Sidebars::MenuItem.new( title: _('Contributors'), link: project_graph_path(context.project, context.current_ref), diff --git a/lib/system_check/app/authorized_keys_permission_check.rb b/lib/system_check/app/authorized_keys_permission_check.rb index 1246a6875a3..9b81812de74 100644 --- a/lib/system_check/app/authorized_keys_permission_check.rb +++ b/lib/system_check/app/authorized_keys_permission_check.rb @@ -19,11 +19,12 @@ module SystemCheck end def show_error - try_fixing_it([ - "sudo chmod 700 #{File.dirname(authorized_keys.file)}", - "touch #{authorized_keys.file}", - "sudo chmod 600 #{authorized_keys.file}" - ]) + try_fixing_it( + [ + "sudo chmod 700 #{File.dirname(authorized_keys.file)}", + "touch #{authorized_keys.file}", + "sudo chmod 600 #{authorized_keys.file}" + ]) fix_and_rerun end diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 76ee5379213..b58d9473794 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -5,28 +5,52 @@ require 'fileutils' module Tasks module Gitlab module Assets - FOSS_ASSET_FOLDERS = %w[app/assets fixtures/emojis vendor/assets/javascripts].freeze + FOSS_ASSET_FOLDERS = %w[app/assets fixtures/emojis vendor/assets].freeze EE_ASSET_FOLDERS = %w[ee/app/assets].freeze JH_ASSET_FOLDERS = %w[jh/app/assets].freeze - JS_ASSET_PATTERNS = %w[*.js config/**/*.js].freeze - JS_ASSET_FILES = %w[package.json yarn.lock].freeze - MASTER_SHA256_HASH_FILE = 'master-assets-hash.txt' - HEAD_SHA256_HASH_FILE = 'assets-hash.txt' - PUBLIC_ASSETS_WEBPACK_DIR = 'public/assets/webpack' + # In the new caching strategy, we check the assets hash sum *before* compiling + # the app/assets/javascripts/locale/**/app.js files. That means the hash sum + # must depend on locale/**/gitlab.po. + JS_ASSET_PATTERNS = %w[*.js config/**/*.js locale/**/gitlab.po].freeze + JS_ASSET_FILES = %w[ + package.json + yarn.lock + babel.config.js + config/webpack.config.js + ].freeze + EXCLUDE_PATTERNS = %w[ + app/assets/javascripts/locale/**/app.js + ].freeze + PUBLIC_ASSETS_DIR = 'public/assets' + HEAD_ASSETS_SHA256_HASH_ENV = 'GITLAB_ASSETS_HASH' + CACHED_ASSETS_SHA256_HASH_FILE = 'cached-assets-hash.txt' + + def self.master_assets_sha256 + @master_assets_sha256 ||= + if File.exist?(Tasks::Gitlab::Assets::CACHED_ASSETS_SHA256_HASH_FILE) + File.read(Tasks::Gitlab::Assets::CACHED_ASSETS_SHA256_HASH_FILE) + else + 'missing!' + end + end + + def self.head_assets_sha256 + @head_assets_sha256 ||= ENV.fetch(Tasks::Gitlab::Assets::HEAD_ASSETS_SHA256_HASH_ENV) do + Tasks::Gitlab::Assets.sha256_of_assets_impacting_compilation(verbose: false) + end + end - def self.sha256_of_assets_impacting_webpack_compilation + def self.sha256_of_assets_impacting_compilation(verbose: true) start_time = Time.now - asset_files = assets_impacting_webpack_compilation - puts "Generating the SHA256 hash for #{assets_impacting_webpack_compilation.size} Webpack-related assets..." + asset_files = assets_impacting_compilation + puts "Generating the SHA256 hash for #{asset_files.size} Webpack-related assets..." if verbose - asset_file_sha256s = asset_files.map do |asset_file| - Digest::SHA256.file(asset_file).hexdigest - end + assets_sha256 = asset_files.map { |asset_file| Digest::SHA256.file(asset_file).hexdigest }.join - Digest::SHA256.hexdigest(asset_file_sha256s.join).tap { |sha256| puts "=> SHA256 generated in #{Time.now - start_time}: #{sha256}" } + Digest::SHA256.hexdigest(assets_sha256).tap { |sha256| puts "=> SHA256 generated in #{Time.now - start_time}: #{sha256}" if verbose } end - def self.assets_impacting_webpack_compilation + def self.assets_impacting_compilation assets_folders = FOSS_ASSET_FOLDERS assets_folders += EE_ASSET_FOLDERS if ::Gitlab.ee? assets_folders += JH_ASSET_FOLDERS if ::Gitlab.jh? @@ -38,52 +62,34 @@ module Tasks asset_files.concat(Dir.glob(["#{folder}/**/*.*"])) end - asset_files + asset_files - Dir.glob(EXCLUDE_PATTERNS) end - - private_class_method :assets_impacting_webpack_compilation + private_class_method :assets_impacting_compilation end end end namespace :gitlab do namespace :assets do + desc 'GitLab | Assets | Return the hash sum of all frontend assets' + task :hash_sum do + print Tasks::Gitlab::Assets.sha256_of_assets_impacting_compilation(verbose: false) + end + desc 'GitLab | Assets | Compile all frontend assets' task :compile do require_dependency 'gitlab/task_helpers' - %w[ - yarn:check - gettext:po_to_json - rake:assets:precompile - gitlab:assets:compile_webpack_if_needed - gitlab:assets:fix_urls - gitlab:assets:check_page_bundle_mixins_css_for_sideeffects - ].each(&::Gitlab::TaskHelpers.method(:invoke_and_time_task)) - end - - desc 'GitLab | Assets | Compile all Webpack assets' - task :compile_webpack_if_needed do - FileUtils.mv(Tasks::Gitlab::Assets::HEAD_SHA256_HASH_FILE, Tasks::Gitlab::Assets::MASTER_SHA256_HASH_FILE, force: true) - - master_assets_sha256 = - if File.exist?(Tasks::Gitlab::Assets::MASTER_SHA256_HASH_FILE) - File.read(Tasks::Gitlab::Assets::MASTER_SHA256_HASH_FILE) - else - 'missing!' - end + puts "Assets SHA256 for `master`: #{Tasks::Gitlab::Assets.master_assets_sha256.inspect}" + puts "Assets SHA256 for `HEAD`: #{Tasks::Gitlab::Assets.head_assets_sha256.inspect}" - head_assets_sha256 = Tasks::Gitlab::Assets.sha256_of_assets_impacting_webpack_compilation.tap do |sha256| - File.write(Tasks::Gitlab::Assets::HEAD_SHA256_HASH_FILE, sha256) - end - - puts "Webpack assets SHA256 for `master`: #{master_assets_sha256}" - puts "Webpack assets SHA256 for `HEAD`: #{head_assets_sha256}" + if Tasks::Gitlab::Assets.head_assets_sha256 != Tasks::Gitlab::Assets.master_assets_sha256 + FileUtils.rm_r(Tasks::Gitlab::Assets::PUBLIC_ASSETS_DIR) if Dir.exist?(Tasks::Gitlab::Assets::PUBLIC_ASSETS_DIR) - public_assets_webpack_dir_exists = Dir.exist?(Tasks::Gitlab::Assets::PUBLIC_ASSETS_WEBPACK_DIR) - - if head_assets_sha256 != master_assets_sha256 || !public_assets_webpack_dir_exists - FileUtils.rm_r(Tasks::Gitlab::Assets::PUBLIC_ASSETS_WEBPACK_DIR) if public_assets_webpack_dir_exists + # gettext:po_to_json needs to run before rake:assets:precompile because + # app/assets/javascripts/locale/**/app.js are pre-compiled by Sprockets + Gitlab::TaskHelpers.invoke_and_time_task('gettext:po_to_json') + Gitlab::TaskHelpers.invoke_and_time_task('rake:assets:precompile') log_path = ENV['WEBPACK_COMPILE_LOG_PATH'] @@ -96,6 +102,9 @@ namespace :gitlab do puts "Written webpack stdout log to #{log_path}" if log_path puts "You can inspect the webpack log here: #{ENV['CI_JOB_URL']}/artifacts/file/#{log_path}" if log_path && ENV['CI_JOB_URL'] + + Gitlab::TaskHelpers.invoke_and_time_task('gitlab:assets:fix_urls') + Gitlab::TaskHelpers.invoke_and_time_task('gitlab:assets:check_page_bundle_mixins_css_for_sideeffects') end end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index ff43a36d930..6647a10898f 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -4,121 +4,168 @@ require 'active_record/fixtures' namespace :gitlab do namespace :backup do + PID = Process.pid.freeze + PID_FILE = "#{Rails.application.root}/tmp/backup_restore.pid" + # Create backup of GitLab system desc 'GitLab | Backup | Create a backup of the GitLab system' task create: :gitlab_environment do - warn_user_is_not_gitlab + lock do + warn_user_is_not_gitlab - Backup::Manager.new(progress).create + Backup::Manager.new(progress).create + end end # Restore backup of GitLab system desc 'GitLab | Backup | Restore a previously created backup' task restore: :gitlab_environment do - warn_user_is_not_gitlab + lock do + warn_user_is_not_gitlab - Backup::Manager.new(progress).restore + Backup::Manager.new(progress).restore + end end namespace :repo do task create: :gitlab_environment do - Backup::Manager.new(progress).run_create_task('repositories') + lock do + Backup::Manager.new(progress).run_create_task('repositories') + end end task restore: :gitlab_environment do - Backup::Manager.new(progress).run_restore_task('repositories') + lock do + Backup::Manager.new(progress).run_restore_task('repositories') + end end end namespace :db do task create: :gitlab_environment do - Backup::Manager.new(progress).run_create_task('main_db') - Backup::Manager.new(progress).run_create_task('ci_db') + lock do + Backup::Manager.new(progress).run_create_task('main_db') + Backup::Manager.new(progress).run_create_task('ci_db') + end end task restore: :gitlab_environment do - Backup::Manager.new(progress).run_restore_task('main_db') - Backup::Manager.new(progress).run_restore_task('ci_db') + lock do + Backup::Manager.new(progress).run_restore_task('main_db') + Backup::Manager.new(progress).run_restore_task('ci_db') + end end end namespace :builds do task create: :gitlab_environment do - Backup::Manager.new(progress).run_create_task('builds') + lock do + Backup::Manager.new(progress).run_create_task('builds') + end end task restore: :gitlab_environment do - Backup::Manager.new(progress).run_restore_task('builds') + lock do + Backup::Manager.new(progress).run_restore_task('builds') + end end end namespace :uploads do task create: :gitlab_environment do - Backup::Manager.new(progress).run_create_task('uploads') + lock do + Backup::Manager.new(progress).run_create_task('uploads') + end end task restore: :gitlab_environment do - Backup::Manager.new(progress).run_restore_task('uploads') + lock do + Backup::Manager.new(progress).run_restore_task('uploads') + end end end namespace :artifacts do task create: :gitlab_environment do - Backup::Manager.new(progress).run_create_task('artifacts') + lock do + Backup::Manager.new(progress).run_create_task('artifacts') + end end task restore: :gitlab_environment do - Backup::Manager.new(progress).run_restore_task('artifacts') + lock do + Backup::Manager.new(progress).run_restore_task('artifacts') + end end end namespace :pages do task create: :gitlab_environment do - Backup::Manager.new(progress).run_create_task('pages') + lock do + Backup::Manager.new(progress).run_create_task('pages') + end end task restore: :gitlab_environment do - Backup::Manager.new(progress).run_restore_task('pages') + lock do + Backup::Manager.new(progress).run_restore_task('pages') + end end end namespace :lfs do task create: :gitlab_environment do - Backup::Manager.new(progress).run_create_task('lfs') + lock do + Backup::Manager.new(progress).run_create_task('lfs') + end end task restore: :gitlab_environment do - Backup::Manager.new(progress).run_restore_task('lfs') + lock do + Backup::Manager.new(progress).run_restore_task('lfs') + end end end namespace :terraform_state do task create: :gitlab_environment do - Backup::Manager.new(progress).run_create_task('terraform_state') + lock do + Backup::Manager.new(progress).run_create_task('terraform_state') + end end task restore: :gitlab_environment do - Backup::Manager.new(progress).run_restore_task('terraform_state') + lock do + Backup::Manager.new(progress).run_restore_task('terraform_state') + end end end namespace :registry do task create: :gitlab_environment do - Backup::Manager.new(progress).run_create_task('registry') + lock do + Backup::Manager.new(progress).run_create_task('registry') + end end task restore: :gitlab_environment do - Backup::Manager.new(progress).run_restore_task('registry') + lock do + Backup::Manager.new(progress).run_restore_task('registry') + end end end namespace :packages do task create: :gitlab_environment do - Backup::Manager.new(progress).run_create_task('packages') + lock do + Backup::Manager.new(progress).run_create_task('packages') + end end task restore: :gitlab_environment do - Backup::Manager.new(progress).run_restore_task('packages') + lock do + Backup::Manager.new(progress).run_restore_task('packages') + end end end @@ -132,6 +179,35 @@ namespace :gitlab do $stdout end end + + def lock + File.open(PID_FILE, File::RDWR | File::CREAT, 0644) do |f| + f.flock(File::LOCK_EX) + + unless f.read.empty? + # There is a PID inside so the process fails + progress.puts(<<~HEREDOC.color(:red)) + Backup and restore in progress: + There is a backup and restore task in progress. Please, try to run the current task once the previous one ends. + If there is no other process running, please remove the PID file manually: rm #{PID_FILE} + HEREDOC + + exit 1 + end + + f.write(PID) + f.flush + ensure + f.flock(File::LOCK_UN) + end + + begin + yield + ensure + progress.puts "#{Time.now} " + "-- Deleting backup and restore lock file".color(:blue) + File.delete(PID_FILE) + end + end end # namespace end: backup end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 30e0e3e72ff..4ef0c396f4a 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -304,14 +304,30 @@ namespace :gitlab do end namespace :migration_testing do - desc 'Run migrations with instrumentation' + # Not possible to import Gitlab::Database::DATABASE_NAMES here + # Specs verify that a task exists for each entry in that array. + all_databases = %i[main ci] + task up: :environment do - Gitlab::Database::Migrations::Runner.up.run + Gitlab::Database::Migrations::Runner.up(database: 'main', legacy_mode: true).run + end + + namespace :up do + all_databases.each do |db| + desc "Run migrations on #{db} with instrumentation" + task db => :environment do + Gitlab::Database::Migrations::Runner.up(database: db).run + end + end end - desc 'Run down migrations in current branch with instrumentation' - task down: :environment do - Gitlab::Database::Migrations::Runner.down.run + namespace :down do + all_databases.each do |db| + desc "Run down migrations on #{db} in current branch with instrumentation" + task db => :environment do + Gitlab::Database::Migrations::Runner.down(database: db).run + end + end end desc 'Sample traditional background migrations with instrumentation' @@ -321,12 +337,24 @@ namespace :gitlab do Gitlab::Database::Migrations::Runner.background_migrations.run_jobs(for_duration: duration) end - desc 'Sample batched background migrations with instrumentation' + namespace :sample_batched_background_migrations do + all_databases.each do |db| + desc "Sample batched background migrations on #{db} with instrumentation" + task db, [:duration_s] => [:environment] do |_t, args| + duration = args[:duration_s]&.to_i&.seconds || 30.minutes # Default of 30 minutes + + Gitlab::Database::Migrations::Runner.batched_background_migrations(for_database: db) + .run_jobs(for_duration: duration) + end + end + end + + desc "Sample batched background migrations with instrumentation (legacy)" task :sample_batched_background_migrations, [:database, :duration_s] => [:environment] do |_t, args| - database_name = args[:database] || 'main' duration = args[:duration_s]&.to_i&.seconds || 30.minutes # Default of 30 minutes - Gitlab::Database::Migrations::Runner.batched_background_migrations(for_database: database_name) + database = args[:database] || 'main' + Gitlab::Database::Migrations::Runner.batched_background_migrations(for_database: database, legacy_mode: true) .run_jobs(for_duration: duration) end end diff --git a/lib/tasks/gitlab/db/lock_writes.rake b/lib/tasks/gitlab/db/lock_writes.rake index eb6d257cac5..421c6a90fdd 100644 --- a/lib/tasks/gitlab/db/lock_writes.rake +++ b/lib/tasks/gitlab/db/lock_writes.rake @@ -14,7 +14,8 @@ namespace :gitlab do table_name: table_name, connection: connection, database_name: database_name, - logger: Logger.new($stdout) + logger: Logger.new($stdout), + dry_run: ENV['DRY_RUN'] == 'true' ) if schemas_for_connection.include?(schema_name.to_sym) diff --git a/lib/tasks/gitlab/openapi.rake b/lib/tasks/gitlab/openapi.rake new file mode 100644 index 00000000000..fd067a1bf0b --- /dev/null +++ b/lib/tasks/gitlab/openapi.rake @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'logger' + +if Rails.env.development? + require 'grape-swagger/rake/oapi_tasks' + GrapeSwagger::Rake::OapiTasks.new('::API::API') +end + +namespace :gitlab do + namespace :openapi do + task :generate do + raise 'This task can only be run in the development environment' unless Rails.env.development? + + ENV['store'] = 'tmp/openapi.json' + Rake::Task["oapi:fetch"].invoke(['openapi.json']) + + yaml_content = Gitlab::Json.parse(File.read('tmp/openapi_swagger_doc.json')).to_yaml + + File.write("doc/api/openapi/openapi_v2.yaml", yaml_content) + end + end +end diff --git a/lib/tasks/gitlab/seed.rake b/lib/tasks/gitlab/seed.rake index 36761165af5..7b9c57b1876 100644 --- a/lib/tasks/gitlab/seed.rake +++ b/lib/tasks/gitlab/seed.rake @@ -35,5 +35,40 @@ namespace :gitlab do puts "\n#{issues_created} issues created!" end end + + task :epics, [:group_full_path, :backfill_weeks, :average_issues_per_week] => :environment do |t, args| + args.with_defaults(backfill_weeks: 5, average_issues_per_week: 2) + + groups = + if args.group_full_path + group = Group.find_by_full_path(args.group_full_path) + + unless group + error_message = "Group '#{args.group_full_path}' does not exist!" + potential_groups = Group.search(args.group_full_path) + + if potential_groups.present? + error_message += " Did you mean '#{potential_groups.first.full_path}'?" + end + + puts error_message.color(:red) + exit 1 + end + + [group] + else + Group.not_mass_generated.find_each + end + + groups.each do |group| + puts "\nSeeding epics for the '#{group.full_path}' group" + seeder = Quality::Seeders::Epics.new(group: group) + epics = seeder.seed( + backfill_weeks: args.backfill_weeks.to_i, + average_issues_per_week: args.average_issues_per_week.to_i + ) + puts "\n#{epics} epics created!" + end + end end end diff --git a/lib/tasks/gitlab/snippets.rake b/lib/tasks/gitlab/snippets.rake index b55f82480e1..d7f71468102 100644 --- a/lib/tasks/gitlab/snippets.rake +++ b/lib/tasks/gitlab/snippets.rake @@ -60,7 +60,7 @@ namespace :gitlab do end def migration_running? - Sidekiq::ScheduledSet.new.any? { |r| r.klass == 'BackgroundMigrationWorker' && r.args[0] == 'BackfillSnippetRepositories' } + Sidekiq::ScheduledSet.new.any? { |r| r.klass == 'BackgroundMigrationWorker' && r.args[0] == 'BackfillSnippetRepositories' } # rubocop:disable Cop/SidekiqApiUsage end # @example diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index 148801254bf..fd9c7114979 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -7,10 +7,10 @@ namespace :tw do task :codeowners do CodeOwnerRule = Struct.new(:category, :writer) DocumentOwnerMapping = Struct.new(:path, :writer) do - def writer_owns_all_pages?(mappings) - mappings - .select { |mapping| mapping.directory == directory } - .all? { |mapping| mapping.writer == writer } + def writer_owns_directory?(mappings) + dir_mappings = mappings.select { |mapping| mapping.directory == directory } + + dir_mappings.count { |mapping| mapping.writer == writer } / dir_mappings.length.to_f > 0.5 end def directory @@ -22,13 +22,12 @@ namespace :tw do CodeOwnerRule.new('Activation', '@phillipwells'), CodeOwnerRule.new('Acquisition', '@phillipwells'), CodeOwnerRule.new('Anti-Abuse', '@phillipwells'), - CodeOwnerRule.new('Authentication and Authorization', '@eread'), + CodeOwnerRule.new('Authentication and Authorization', '@jglassman1'), CodeOwnerRule.new('Certify', '@msedlakjakubowski'), CodeOwnerRule.new('Code Review', '@aqualls'), CodeOwnerRule.new('Compliance', '@eread'), CodeOwnerRule.new('Composition Analysis', '@rdickenson'), CodeOwnerRule.new('Configure', '@phillipwells'), - CodeOwnerRule.new('Container Security', '@claytoncornell'), CodeOwnerRule.new('Contributor Experience', '@eread'), CodeOwnerRule.new('Conversion', '@kpaizee'), CodeOwnerRule.new('Database', '@aqualls'), @@ -39,7 +38,7 @@ namespace :tw do CodeOwnerRule.new('Documentation Guidelines', '@sselhorn'), CodeOwnerRule.new('Dynamic Analysis', '@rdickenson'), CodeOwnerRule.new('Ecosystem', '@kpaizee'), - CodeOwnerRule.new('Editor', '@aqualls'), + CodeOwnerRule.new('Editor', '@ashrafkhamis'), CodeOwnerRule.new('Foundations', '@rdickenson'), CodeOwnerRule.new('Fuzz Testing', '@rdickenson'), CodeOwnerRule.new('Geo', '@axil'), @@ -58,6 +57,7 @@ namespace :tw do CodeOwnerRule.new('Pipeline Execution', '@marcel.amirault'), CodeOwnerRule.new('Pipeline Insights', '@marcel.amirault'), CodeOwnerRule.new('Portfolio Management', '@msedlakjakubowski'), + CodeOwnerRule.new('Product Analytics', '@lciutacu'), CodeOwnerRule.new('Product Intelligence', '@claytoncornell'), CodeOwnerRule.new('Product Planning', '@msedlakjakubowski'), CodeOwnerRule.new('Project Management', '@msedlakjakubowski'), @@ -68,6 +68,7 @@ namespace :tw do CodeOwnerRule.new('Respond', '@msedlakjakubowski'), CodeOwnerRule.new('Runner', '@sselhorn'), CodeOwnerRule.new('Pods', '@sselhorn'), + CodeOwnerRule.new('Security Policies', '@claytoncornell'), CodeOwnerRule.new('Source Code', '@aqualls'), CodeOwnerRule.new('Static Analysis', '@rdickenson'), CodeOwnerRule.new('Style Guide', '@sselhorn'), @@ -114,14 +115,14 @@ namespace :tw do deduplicated_mappings = Set.new mappings.each do |mapping| - if mapping.writer_owns_all_pages?(mappings) + if mapping.writer_owns_directory?(mappings) deduplicated_mappings.add("#{mapping.directory}/ #{mapping.writer}") else deduplicated_mappings.add("#{mapping.path} #{mapping.writer}") end end - deduplicated_mappings.each { |mapping| puts mapping } + deduplicated_mappings.sort.each { |mapping| puts mapping } if errors.present? puts "-----" diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index 73a79427da3..159b70cd673 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -43,17 +43,10 @@ namespace :gitlab do # Do not edit it manually! BANNER - repository_includes = ci_template_includes_hash(:repository_source) - auto_devops_jobs_includes = ci_template_includes_hash(:auto_devops_source, 'Jobs') - auto_devops_security_includes = ci_template_includes_hash(:auto_devops_source, 'Security') - all_includes = [ - *repository_includes, - ci_template_event('p_ci_templates_implicit_auto_devops'), - *auto_devops_jobs_includes, - *auto_devops_security_includes - ] - - File.write(Gitlab::UsageDataCounters::CiTemplateUniqueCounter::KNOWN_EVENTS_FILE_PATH, banner + YAML.dump(all_includes).gsub(/ *$/m, '')) + all_includes = explicit_template_includes + implicit_auto_devops_includes + yaml = banner + YAML.dump(all_includes).gsub(/ *$/m, '') + + File.write(Gitlab::UsageDataCounters::CiTemplateUniqueCounter::KNOWN_EVENTS_FILE_PATH, yaml) end desc 'GitLab | UsageDataMetrics | Generate raw SQL metrics queries for RSpec' @@ -65,16 +58,27 @@ namespace :gitlab do end FileUtils.mkdir_p(path) - FileUtils.chdir(path) - File.write('sql_metrics_queries.json', Gitlab::Json.pretty_generate(queries)) + File.write(File.join(path, 'sql_metrics_queries.json'), Gitlab::Json.pretty_generate(queries)) + end + + # Events for templates included via YAML-less Auto-DevOps + def implicit_auto_devops_includes + Gitlab::UsageDataCounters::CiTemplateUniqueCounter + .all_included_templates('Auto-DevOps.gitlab-ci.yml') + .map { |template| implicit_auto_devops_event(template) } + .uniq + .sort_by { _1['name'] } end - def ci_template_includes_hash(source, template_directory = nil) - Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_templates("lib/gitlab/ci/templates/#{template_directory}").map do |template| - expanded_template_name = Gitlab::UsageDataCounters::CiTemplateUniqueCounter.expand_template_name("#{template_directory}/#{template}") - event_name = Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_template_event_name(expanded_template_name, source) + # Events for templates included in a .gitlab-ci.yml using include:template + def explicit_template_includes + Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_templates("lib/gitlab/ci/templates/").each_with_object([]) do |template, result| + expanded_template_name = Gitlab::UsageDataCounters::CiTemplateUniqueCounter.expand_template_name(template) + next unless expanded_template_name # guard against templates unavailable on FOSS - ci_template_event(event_name) + event_name = Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_template_event_name(expanded_template_name, :repository_source) + + result << ci_template_event(event_name) end end @@ -86,5 +90,10 @@ namespace :gitlab do 'aggregation' => 'weekly' } end + + def implicit_auto_devops_event(expanded_template_name) + event_name = Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_template_event_name(expanded_template_name, :auto_devops_source) + ci_template_event(event_name) + end end end diff --git a/lib/version_check.rb b/lib/version_check.rb index 2d132001f54..35014f3ddf0 100644 --- a/lib/version_check.rb +++ b/lib/version_check.rb @@ -5,6 +5,36 @@ require "base64" class VersionCheck include ReactiveCaching + ## Version Check Reactive Caching + ## This cache stores the external API response from https://version.gitlab.com + ## + ## Example API Response + ## { + ## "latest_version": "15.2.2", + ## "severity": "success" + ## } + ## + ## This response from this endpoint only changes in 2 scenarios: + ## 1. Customer upgrades their GitLab Instance + ## 2. GitLab releases a new version + ## + ## We use GitLab::VERSION as the identifier for the cached information. + ## This means if the user upgrades their version we will create a new cache record. + ## The old one will be invalidated and cleaned up at the end of the self.reactive_cache_lifetime. + ## + ## - self.reactive_cache_refresh_interval = 12.hours + ## We want to prevent as many external API calls as possible to save on resources. + ## Since an EXISTING cache record will only become "invalid" if GitLab releases a new version we + ## determined that 12 hour intervals is enough of a window to capture an available upgrade. + ## + ## - self.reactive_cache_lifetime = 7.days + ## We don't want the data to be missing every time a user revisits a page using this info. + ## Thus 7 days seems like a fair amount of time before we erase the cache. + ## This also will handle cleaning up old cache records as they will no longer be accessed after an upgrade. + ## + + self.reactive_cache_refresh_interval = 12.hours + self.reactive_cache_lifetime = 7.days self.reactive_cache_work_type = :external_dependency self.reactive_cache_worker_finder = ->(_id, *args) { from_cache } |