diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-04-26 15:17:18 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-04-26 15:17:18 +0000 |
| commit | 34283a71d9ac31eb4da0b59d0b25fc2be014bc9c (patch) | |
| tree | 4a780bf8d9c57c3b8ce34fc81152d8a87f71d594 | |
| parent | 9edf852c3a851d84b85bc94f7a3b41d5ef04dd32 (diff) | |
| download | gitlab-ce-34283a71d9ac31eb4da0b59d0b25fc2be014bc9c.tar.gz | |
Add latest changes from gitlab-org/gitlab@master
75 files changed, 8137 insertions, 379 deletions
diff --git a/README.md b/README.md index b61af1b1f42..6455e7d458e 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ We are hiring developers, support people, and production engineers all the time, On [about.gitlab.com](https://about.gitlab.com/) you can find more information about: - [Subscriptions](https://about.gitlab.com/pricing/) -- [Consultancy](https://about.gitlab.com/consultancy/) +- [Professional Services](https://about.gitlab.com/services/) - [Community](https://about.gitlab.com/community/) - [Hosted GitLab.com](https://about.gitlab.com/gitlab-com/) use GitLab as a free service - [GitLab Enterprise Edition](https://about.gitlab.com/features/#enterprise) with additional features aimed at larger organizations. diff --git a/app/assets/stylesheets/themes/dark_mode_overrides.scss b/app/assets/stylesheets/themes/dark_mode_overrides.scss index 89a49b2cf86..619336686e8 100644 --- a/app/assets/stylesheets/themes/dark_mode_overrides.scss +++ b/app/assets/stylesheets/themes/dark_mode_overrides.scss @@ -300,3 +300,8 @@ body.gl-dark { // soften on darkmode background-color: mix($gray-50, $orange-50, 75%) !important; } + +.tanuki-bot-chat-drawer .tanuki-bot-message { + // lightens chat bubble in darkmode as $gray-50 matches drawer background. See tanuki_bot_chat.scss + background-color: $gray-100; +} diff --git a/app/controllers/jira_connect/subscriptions_controller.rb b/app/controllers/jira_connect/subscriptions_controller.rb index a206e7fbbd8..76f0946762a 100644 --- a/app/controllers/jira_connect/subscriptions_controller.rb +++ b/app/controllers/jira_connect/subscriptions_controller.rb @@ -19,10 +19,6 @@ class JiraConnect::SubscriptionsController < JiraConnect::ApplicationController p.style_src(*style_src_values) end - before_action do - push_frontend_feature_flag(:jira_connect_oauth, @user) - end - before_action :allow_rendering_in_iframe, only: :index before_action :verify_qsh_claim!, only: :index before_action :allow_self_managed_content_security_policy, only: :index diff --git a/app/graphql/types/release_asset_link_type.rb b/app/graphql/types/release_asset_link_type.rb index b2bc52c7745..c1669f67666 100644 --- a/app/graphql/types/release_asset_link_type.rb +++ b/app/graphql/types/release_asset_link_type.rb @@ -9,12 +9,6 @@ module Types present_using Releases::LinkPresenter - field :external, GraphQL::Types::Boolean, - null: true, - method: :external?, - description: 'Indicates the link points to an external resource.', - deprecated: { reason: 'No longer used', milestone: '15.9' } - field :id, GraphQL::Types::ID, null: false, description: 'ID of the link.' field :link_type, diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 42c9481828c..a7694c58588 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -484,7 +484,8 @@ module ApplicationSettingsHelper :user_defaults_to_private_profile, :deactivation_email_additional_text, :projects_api_rate_limit_unauthenticated, - :gitlab_dedicated_instance + :gitlab_dedicated_instance, + :ci_max_includes ].tap do |settings| next if Gitlab.com? diff --git a/app/helpers/jira_connect_helper.rb b/app/helpers/jira_connect_helper.rb index 28b30ae051c..af2c09554f0 100644 --- a/app/helpers/jira_connect_helper.rb +++ b/app/helpers/jira_connect_helper.rb @@ -11,7 +11,7 @@ module JiraConnectHelper subscriptions_path: jira_connect_subscriptions_path(format: :json), users_path: current_user ? nil : jira_connect_users_path, # users_path is used to determine if user is signed in gitlab_user_path: current_user ? user_path(current_user) : nil, - oauth_metadata: Feature.enabled?(:jira_connect_oauth, current_user) ? jira_connect_oauth_data(installation).to_json : nil, + oauth_metadata: jira_connect_oauth_data(installation).to_json, public_key_storage_enabled: Gitlab::CurrentSettings.jira_connect_public_key_storage_enabled? } end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 78668d25c63..4e14aa56be1 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -388,6 +388,8 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord validates :max_yaml_size_bytes, numericality: { only_integer: true, greater_than: 0 }, presence: true validates :max_yaml_depth, numericality: { only_integer: true, greater_than: 0 }, presence: true + validates :ci_max_includes, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, presence: true + validates :email_restrictions, untrusted_regexp: true validates :hashed_storage_enabled, inclusion: { in: [true], message: N_("Hashed storage can't be disabled anymore for new projects") } diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 010c88179df..845d402f550 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -252,7 +252,8 @@ module ApplicationSettingImplementation allow_runner_registration_token: true, user_defaults_to_private_profile: false, projects_api_rate_limit_unauthenticated: 400, - gitlab_dedicated_instance: false + gitlab_dedicated_instance: false, + ci_max_includes: 150 }.tap do |hsh| hsh.merge!(non_production_defaults) unless Rails.env.production? end diff --git a/app/views/admin/application_settings/_ci_cd.html.haml b/app/views/admin/application_settings/_ci_cd.html.haml index f162f5be746..0c9d5a5a8df 100644 --- a/app/views/admin/application_settings/_ci_cd.html.haml +++ b/app/views/admin/application_settings/_ci_cd.html.haml @@ -46,6 +46,11 @@ .form-group = f.gitlab_ui_checkbox_component :protected_ci_variables, s_('AdminSettings|Protect CI/CD variables by default'), help_text: s_('AdminSettings|New CI/CD variables in projects and groups default to protected.') .form-group + = f.label :ci_max_includes, s_('AdminSettings|Maximum includes'), class: 'label-bold' + = f.number_field :ci_max_includes, class: 'form-control gl-form-input' + .form-text.text-muted + = s_('AdminSettings|The maximum number of included files per pipeline.') + .form-group = f.label :ci_config_path, _('Default CI/CD configuration file'), class: 'label-bold' = f.text_field :default_ci_config_path, class: 'form-control gl-form-input', placeholder: '.gitlab-ci.yml' %p.form-text.text-muted diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 446cba267ad..387f817ac5e 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1245,15 +1245,6 @@ :weight: 1 :idempotent: false :tags: [] -- :name: github_importer:github_import_import_release_attachments - :worker_name: Gitlab::GithubImport::ImportReleaseAttachmentsWorker - :feature_category: :importers - :has_external_dependencies: true - :urgency: :low - :resource_boundary: :unknown - :weight: 1 - :idempotent: false - :tags: [] - :name: github_importer:github_import_pull_requests_import_review_request :worker_name: Gitlab::GithubImport::PullRequests::ImportReviewRequestWorker :feature_category: :importers diff --git a/app/workers/gitlab/github_import/import_release_attachments_worker.rb b/app/workers/gitlab/github_import/import_release_attachments_worker.rb deleted file mode 100644 index bf901f2f7b8..00000000000 --- a/app/workers/gitlab/github_import/import_release_attachments_worker.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -# TODO: remove in 16.X milestone -# https://gitlab.com/gitlab-org/gitlab/-/issues/377059 -module Gitlab - module GithubImport - class ImportReleaseAttachmentsWorker # rubocop:disable Scalability/IdempotentWorker - include ObjectImporter - - def representation_class - Representation::NoteText - end - - def importer_class - Importer::NoteAttachmentsImporter - end - - def object_type - :release_attachment - end - end - end -end diff --git a/config/feature_flags/development/jira_connect_oauth.yml b/config/feature_flags/development/graphql_subs_lb.yml index 2f090d56bd1..e3fa9b02e12 100644 --- a/config/feature_flags/development/jira_connect_oauth.yml +++ b/config/feature_flags/development/graphql_subs_lb.yml @@ -1,8 +1,8 @@ --- -name: jira_connect_oauth -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81126 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/355048 -milestone: '14.9' +name: graphql_subs_lb +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117678 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/408178 +milestone: '16.0' type: development -group: group::integrations +group: group::application performance default_enabled: false diff --git a/db/migrate/20230419130952_remove_github_import_job_instances.rb b/db/migrate/20230419130952_remove_github_import_job_instances.rb new file mode 100644 index 00000000000..3e1ccd3c74d --- /dev/null +++ b/db/migrate/20230419130952_remove_github_import_job_instances.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class RemoveGithubImportJobInstances < Gitlab::Database::Migration[2.1] + DEPRECATED_JOB_CLASSES = %w[ + Gitlab::GithubImport::ImportReleaseAttachmentsWorker + ] + + def up + sidekiq_remove_jobs(job_klasses: DEPRECATED_JOB_CLASSES) + end + + def down + # This migration removes any instances of deprecated workers and cannot be undone. + end +end diff --git a/db/migrate/20230421124302_epic_boards_show_colors.rb b/db/migrate/20230421124302_epic_boards_show_colors.rb new file mode 100644 index 00000000000..cfc8e228196 --- /dev/null +++ b/db/migrate/20230421124302_epic_boards_show_colors.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class EpicBoardsShowColors < Gitlab::Database::Migration[2.1] + def change + add_column :boards_epic_boards, :display_colors, :boolean, default: true, null: false + end +end diff --git a/db/migrate/20230421165020_add_ci_max_includes_to_application_settings.rb b/db/migrate/20230421165020_add_ci_max_includes_to_application_settings.rb new file mode 100644 index 00000000000..9996d3b1654 --- /dev/null +++ b/db/migrate/20230421165020_add_ci_max_includes_to_application_settings.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class AddCiMaxIncludesToApplicationSettings < Gitlab::Database::Migration[2.1] + def change + add_column :application_settings, :ci_max_includes, :integer, default: 150, null: false + end +end diff --git a/db/schema_migrations/20230419130952 b/db/schema_migrations/20230419130952 new file mode 100644 index 00000000000..fe1a371aad9 --- /dev/null +++ b/db/schema_migrations/20230419130952 @@ -0,0 +1 @@ +1172667d682c57cae72696030f3e5f58a57f5c71244190785a02c07e065173c6
\ No newline at end of file diff --git a/db/schema_migrations/20230421124302 b/db/schema_migrations/20230421124302 new file mode 100644 index 00000000000..0cbe24b1e6d --- /dev/null +++ b/db/schema_migrations/20230421124302 @@ -0,0 +1 @@ +3d8630d52fa7998ffd4775d0e070e778c8317416b241fd6a5149188ca7d2ff35
\ No newline at end of file diff --git a/db/schema_migrations/20230421165020 b/db/schema_migrations/20230421165020 new file mode 100644 index 00000000000..a1121e3f83d --- /dev/null +++ b/db/schema_migrations/20230421165020 @@ -0,0 +1 @@ +1e33e8da1f2f10f976e19bcf234f7ac5bdcc0eed7d60288ead2eaf87dfd678c9
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index c366c1e3bb6..8b0042bfbdf 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -11804,6 +11804,7 @@ CREATE TABLE application_settings ( encrypted_product_analytics_configurator_connection_string_iv bytea, silent_mode_enabled boolean DEFAULT false NOT NULL, package_metadata_purl_types smallint[] DEFAULT '{}'::smallint[], + ci_max_includes integer DEFAULT 150 NOT NULL, CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)), CONSTRAINT app_settings_container_registry_pre_import_tags_rate_positive CHECK ((container_registry_pre_import_tags_rate >= (0)::numeric)), CONSTRAINT app_settings_dep_proxy_ttl_policies_worker_capacity_positive CHECK ((dependency_proxy_ttl_group_policy_worker_capacity >= 0)), @@ -12596,6 +12597,7 @@ CREATE TABLE boards_epic_boards ( created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, name text DEFAULT 'Development'::text NOT NULL, + display_colors boolean DEFAULT true NOT NULL, CONSTRAINT check_bcbbffe601 CHECK ((char_length(name) <= 255)) ); diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index c8272c1a90d..791c6c2677a 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -973,6 +973,7 @@ Input type: `AiActionInput` | <a id="mutationaiactionclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationaiactionexplaincode"></a>`explainCode` | [`AiExplainCodeInput`](#aiexplaincodeinput) | Input for explain_code AI action. | | <a id="mutationaiactionexplainvulnerability"></a>`explainVulnerability` | [`AiExplainVulnerabilityInput`](#aiexplainvulnerabilityinput) | Input for explain_vulnerability AI action. | +| <a id="mutationaiactionmarkupformat"></a>`markupFormat` | [`MarkupFormat`](#markupformat) | Indicates the response format. | | <a id="mutationaiactionsummarizecomments"></a>`summarizeComments` | [`AiSummarizeCommentsInput`](#aisummarizecommentsinput) | Input for summarize_comments AI action. | #### Fields @@ -3061,6 +3062,7 @@ Input type: `EpicBoardCreateInput` | Name | Type | Description | | ---- | ---- | ----------- | | <a id="mutationepicboardcreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationepicboardcreatedisplaycolors"></a>`displayColors` | [`Boolean`](#boolean) | Whether or not display epic colors. Ignored unless `epic_color_highlight` flag is enabled. | | <a id="mutationepicboardcreategrouppath"></a>`groupPath` | [`ID`](#id) | Full path of the group with which the resource is associated. | | <a id="mutationepicboardcreatehidebackloglist"></a>`hideBacklogList` | [`Boolean`](#boolean) | Whether or not backlog list is hidden. | | <a id="mutationepicboardcreatehideclosedlist"></a>`hideClosedList` | [`Boolean`](#boolean) | Whether or not closed list is hidden. | @@ -3127,6 +3129,7 @@ Input type: `EpicBoardUpdateInput` | Name | Type | Description | | ---- | ---- | ----------- | | <a id="mutationepicboardupdateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationepicboardupdatedisplaycolors"></a>`displayColors` | [`Boolean`](#boolean) | Whether or not display epic colors. Ignored unless `epic_color_highlight` flag is enabled. | | <a id="mutationepicboardupdatehidebackloglist"></a>`hideBacklogList` | [`Boolean`](#boolean) | Whether or not backlog list is hidden. | | <a id="mutationepicboardupdatehideclosedlist"></a>`hideClosedList` | [`Boolean`](#boolean) | Whether or not closed list is hidden. | | <a id="mutationepicboardupdateid"></a>`id` | [`BoardsEpicBoardID!`](#boardsepicboardid) | Epic board global ID. | @@ -13951,6 +13954,7 @@ Represents an epic board. | Name | Type | Description | | ---- | ---- | ----------- | +| <a id="epicboarddisplaycolors"></a>`displayColors` | [`Boolean`](#boolean) | Whether or not display epic colors. | | <a id="epicboardhidebackloglist"></a>`hideBacklogList` | [`Boolean`](#boolean) | Whether or not backlog list is hidden. | | <a id="epicboardhideclosedlist"></a>`hideClosedList` | [`Boolean`](#boolean) | Whether or not closed list is hidden. | | <a id="epicboardid"></a>`id` | [`BoardsEpicBoardID!`](#boardsepicboardid) | Global ID of the epic board. | @@ -20424,7 +20428,6 @@ Represents an asset link associated with a release. | ---- | ---- | ----------- | | <a id="releaseassetlinkdirectassetpath"></a>`directAssetPath` | [`String`](#string) | Relative path for the direct asset link. | | <a id="releaseassetlinkdirectasseturl"></a>`directAssetUrl` | [`String`](#string) | Direct asset URL of the link. | -| <a id="releaseassetlinkexternal"></a>`external` **{warning-solid}** | [`Boolean`](#boolean) | **Deprecated** in 15.9. No longer used. | | <a id="releaseassetlinkid"></a>`id` | [`ID!`](#id) | ID of the link. | | <a id="releaseassetlinklinktype"></a>`linkType` | [`ReleaseAssetLinkType`](#releaseassetlinktype) | Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other`. | | <a id="releaseassetlinkname"></a>`name` | [`String`](#string) | Name of the link. | @@ -24149,6 +24152,16 @@ List limit metric setting. | <a id="listlimitmetricissue_count"></a>`issue_count` | Limit list by number of issues. | | <a id="listlimitmetricissue_weights"></a>`issue_weights` | Limit list by total weight of issues. | +### `MarkupFormat` + +List markup formats. + +| Value | Description | +| ----- | ----------- | +| <a id="markupformathtml"></a>`HTML` | HTML format. | +| <a id="markupformatmarkdown"></a>`MARKDOWN` | Markdown format. | +| <a id="markupformatraw"></a>`RAW` | Raw format. | + ### `MeasurementIdentifier` Possible identifier types for a measurement. diff --git a/doc/api/graphql/removed_items.md b/doc/api/graphql/removed_items.md index 57f1c49290f..d242c889290 100644 --- a/doc/api/graphql/removed_items.md +++ b/doc/api/graphql/removed_items.md @@ -10,6 +10,18 @@ GraphQL is a versionless API, unlike the REST API. Occasionally, items have to be updated or removed from the GraphQL API. According to our [process for removing items](index.md#deprecation-and-removal-process), here are the items that have been removed. +## GitLab 16.0 + +Fields removed in GitLab 16.0. + +### GraphQL Fields + +[Removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/111750) in GitLab 16.0: + +| Field name | GraphQL type | Deprecated in | Use instead | +|----------------------|--------------------------|---------------|----------------------------| +| `external` | `GraphQL::Types::Boolean`| 15.9 | None | + ## GitLab 15.0 Fields removed in GitLab 15.0. diff --git a/doc/api/jobs.md b/doc/api/jobs.md index 19d28420069..f060d86c335 100644 --- a/doc/api/jobs.md +++ b/doc/api/jobs.md @@ -867,8 +867,9 @@ POST /projects/:id/jobs/:job_id/play Example request: ```shell -curl --request POST "https://gitlab.example.com/api/v4/projects/1/jobs/1/play - --header "PRIVATE-TOKEN: <your_access_token>" +curl --request POST "https://gitlab.example.com/api/v4/projects/1/jobs/1/play" \ + --header "Content-Type: application/json" \ + --header "PRIVATE-TOKEN: <your_access_token>" \ --data @variables.json ``` diff --git a/doc/api/settings.md b/doc/api/settings.md index 22e21ba19b4..9c8c62ed223 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -296,6 +296,7 @@ listed in the descriptions of the relevant settings. | `bulk_import_enabled` | boolean | no | Enable migrating GitLab groups by direct transfer. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/383268) in GitLab 15.8. Setting also [available](../user/admin_area/settings/visibility_and_access_controls.md#enable-migration-of-groups-and-projects-by-direct-transfer) in the Admin Area. | | `can_create_group` | boolean | no | Indicates whether users can create top-level groups. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367754) in GitLab 15.5. Defaults to `true`. | | `check_namespace_plan` **(PREMIUM)** | boolean | no | Enabling this makes only licensed EE features available to projects if the project namespace's plan includes the feature or if the project is public. | +| `ci_max_includes` | integer | no | The maximum number of [includes](../ci/yaml/includes.md) per pipeline. Default is `150`. | | `commit_email_hostname` | string | no | Custom hostname (for private commit emails). | | `container_expiration_policies_enable_historic_entries` | boolean | no | Enable [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#enable-the-cleanup-policy) for all projects. | | `container_registry_cleanup_tags_service_max_list_size` | integer | no | The maximum number of tags that can be deleted in a single execution of [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#set-cleanup-limits-to-conserve-resources). | diff --git a/doc/architecture/blueprints/gitlab_ml_experiments/index.md b/doc/architecture/blueprints/gitlab_ml_experiments/index.md new file mode 100644 index 00000000000..f912a7092de --- /dev/null +++ b/doc/architecture/blueprints/gitlab_ml_experiments/index.md @@ -0,0 +1,172 @@ +--- +status: proposed +creation-date: "2023-04-13" +authors: [ "@andrewn" ] +coach: "@grzesiek" +--- + +# GitLab Service-Integration: AI and Beyond + +[[_TOC_]] + +This document is an abbreviated proposal for Service-Integration to allow teams within GitLab to rapidly build new application features that leverage AI, ML, and data technologies. + +## Executive Summary + +This document proposes a service-integration approach to setting up infrastructure to allow teams within GitLab to build new application features that leverage AI, ML, and data technologies at a rapid pace. The scope of the document is limited specifically to internally hosted features, not third-party APIs. The current application architecture runs most GitLab application features in Ruby. However, many ML/AI experiments require different resources and tools, implemented in different languages, with huge libraries that do not always play nicely together, and have different hardware requirements. Adding all these features to the existing infrastructure will increase the size of the GitLab application container rapidly, resulting in slower startup times, increased number of dependencies, security risks, negatively impacting development velocity, and increasing complexity due to different hardware requirements. As an alternative, the proposal suggests adding services to avoid overloading GitLabs main workloads. These services will run independently with isolated resources and dependencies. By adding services, GitLab can maintain the availability and security of GitLab.com, and enable engineers to rapidly iterate on new ML/AI experiments. + +## Scope + +The infrastructure, platform, and other changes related to ML/AI experiments is broad. This blueprint is limited specifically to the following scope: + +1. Production workloads, running (directly or indirectly) as a result of requests into the GitLab application (`gitlab.com`), or an associated subdomains (for example, `codesuggestions.gitlab.com`). +1. Excludes requests from the GitLab application, made to third-party APIs outside of our infrastructure. From an Infrastructure point-of-view, external AI/ML API requests are no different from other API (non ML/AI) requests and generally follow the existing guidelines that are in place for calling external APIs. +1. Excludes training and tuning workloads not _directly_ connected to our production workloads. Training and tuning workloads are distinct from production workloads and will be covered by their own blueprint(s). + +## Running Production ML/AI experiment workloads + +### Why Not Simply Continue To Use The Existing Application Architecture? + +Let's start with some background on how the application is deployed: + +1. Most GitLab application features are implemented in Ruby and run in one of two types of Ruby deployments: broadly Rails and Sidekiq (although we do partition this traffic further for different workloads). +1. These Ruby workloads have two main container images `gitlab-webservice-ee` and `gitlab-sidekiq-ee`. All the code, libraries, binaries, and other resources that we use to support the main Ruby part of the codebase are embedded within these images. +1. There are thousands of pods running these containers in production for GitLab.com at any moment in time. They are started up and shut down at a high rate throughout the day as traffic demands on the site fluctuate. +1. For _most_ new features developed, any new supporting resources need to be added to either one, or both of these containers. + +\ +[source](https://docs.google.com/drawings/d/1RiTUnsDSkTGaMqK_RfUlCd_rQ6CgSInhfQJNewIKf1M/edit) + +Many of the initial discussions focus on adding supporting resources to these existing containers ([example](https://gitlab.com/gitlab-org/gitlab/-/issues/403630#note_1345192671)). Choosing this approach would have many downsides, in terms of both the velocity at which new features can be iterated on, and in terms of the availability of GitLab.com. + +Many of the AI experiments that GitLab is considering integrating into the application are substantially different from other libraries and tools that have been integrated in the past. + +1. ML toolkits are **implemented in a plethora of languages**, each requiring separate runtimes. Python, C, C++ are the most common, but there is a long tail of languages used. +1. There are a very large number of tools that we're looking to integrate with and **no single tool will support all the features that are being investigated**. Tensorflow, PyTorch, Keras, Scikit-learn, Alpaca are just a few examples. +1. **These libraries are huge**. Tensorflow's container image with GPU support is 3GB, PyTorch is 5GB, Keras is 300MB. Prophet is ~250MB. +1. Many of these **libraries do not play nicely together**: they may have dependencies that are not compatible, or require different versions of Python, or GPU driver versions. + +It's likely that in the next few months, GitLab will experiment with many different features, using many different libraries. + +Trying to deploy all of these features into the existing infrastructure would have many downsides: + +1. **The size of the GitLab application container would expand very rapidly** as each new experiment introduces a new set of supporting libraries, each library is as big, or bigger, than the existing GitLab application within the container. +1. **Startup times for new workloads would increase**, potentially impacting the availability of GitLab.com during high-traffic periods. +1. The number of dependencies within the container would increase rapidly, putting pressure on the engineering teams to **keep ahead of exploits and vulnerabilities**. +1. **The security attack surface within the container would be greatly increased** with each new dependency. These containers include secrets which, if leaked via an exploit would need costly application-wide secret rotation to be done. +1. **Development velocity will be negatively impacted** as engineers work to avoid dependency conflicts between libraries. +1. Additionally there may be **extra complexity due to different hardware requirements** for different libraries with appropriate drivers etc for GPUs, TPUs, CUDA versions, etc. +1. Our Kubernetes workloads have been tuned for the existing multithreaded Ruby request (Rails) and message (Sidekiq) processes. Adding extremely resource-intensive applications into these workloads would affect unrelated requests, **starving requests of CPU and memory and requiring complex tuning to ensure fairness**. Failure to do this would impact our availability of GitLab.com. + + +\ +[source](https://docs.google.com/drawings/d/1aYffBzzea5QuZ-mTMteowefbV7VmsOuq2v4BqbPd6KE/edit) + +### Proposal: Avoid Overfilling GitLabs Application Containers with Service-Integration + +GitLab.com migrated to Kubernetes several years back, but for numerous good reasons, the application architecture deployed for GitLab.com remains fairly simple. + +Instead of embedding these applications directly into the Rails and/or Sidekiq containers, we run them as small, independent Kubernetes deployments, isolated from the main workload. + +\ +[source](https://docs.google.com/drawings/d/1ZPprcSYH5Oqp8T46I0p1Hhr-GD55iREDvFWcpQq9dTQ/edit) + +The service-integration approach has already been used for the [Suggested Reviewers feature](https://gitlab.com/gitlab-com/gl-infra/readiness/-/merge_requests/114) that has been deployed to GitLab.com. + +This approach would have many advantages: + +1. **Componentization and Replaceability**: some of these AI feature experiments will likely be short-lived. Being able to shut them down (possibly quickly, in an emergency, such as a security breach) is important. If they are terminated, they are less likely to leave technical debt behind in our main application workloads. +1. **Security Isolation**: experimental services can run with access to a minimal set of secrets, or possibly none. Ideally, the services would be stateless, with data being passed in, processed, and returned to the caller without access to PostgreSQL or other data sources. In the event of a remote code exploit or other security breach, the attacker would have limited access to sensitive data. + 1. In lieu of direct access to the main or CI Postgres clusters, services would be provided with access to the internal GitLab API through a predefined internal URL. The platform should provide instrumentation and monitoring on this address. + 1. In future iterations, but out of scope for the initial delivery, the platform could facilitate automatic authentication against the internal API, for example by managing and injecting short-lived API tokens into internal API calls, or OIDC etc. +1. **Resource Isolation**: resource-intensive workloads would be isolated to individual containers. OOM failures would not impact requests outside of the experiment. CPU saturation would not slow down unrelated requests. +1. **Dependency Isolation**: different AI libraries will have conflicting dependencies. This will not be an issue if they're run as separate services in Kubernetes. +1. **Container Size**: the size of the main application containers is not drastically increased, placing a burden on the application. +1. **Distribution Team Bottleneck**: The Distribution team avoids becoming a bottleneck as demands for many different libraries to be included in the main application containers increase. +1. **Stronger Ownership of Workloads**: teams can better understand how their workloads are running as they run in isolation. + +However, there are several outstanding questions: + +1. **Availability Requirements**: would experimental services have the same availability requirements (and alerting requirements) as the main application? +1. **Oncall**: would teams be responsible for handling pager alerts for their services? +1. **Support for non-SAAS GitLab instances**: initially all experiments would target GitLab.com, but eventually we may need to consider how to support other instances. + 1. There are three possible modes for services: + 1. `M1`: GitLab.com only: only GitLab.com supports the service. + 1. `M2`: SAAS-hosted for use with self-managed instance and instance-hosted: a singular SAAS-hosted service supports self-managed instances and GitLab.com. This is similar to the [GitLab Plus proposal](https://gitlab.com/groups/gitlab-org/-/epics/308). + 1. `M3`: Instance-hosted: each instance has a copy of the service. GitLab.com has a copy for GitLab.com. Self-managed instances host their copy of the service. This is similar to the container registry or Gitaly today. + 1. Initially, most experiments will probably be option 1 but may be promoted to 2 or 3 as they mature. +1. **Promotion Process**: ML/AI experimental features will need to be promoted to non-experimental status as they mature. A process for this will need to be established. + +#### Proposed Guidelines for Building ML/AI Services + +1. Avoid adding any large ML/AI libraries needed to support experimentation to the main application. +1. Create an platform to support individual ML/AI experiments. +1. Encourage supporting services to be stateless (excluding deployed models and other resources generated during ML training). +1. ML/AI experiment support services must not access main application datastores, including but not limited to main PostgreSQL, CI PostgreSQL, and main application Redis instances. +1. In the main application, client code for services should reside behind a feature-flag toggle, for fine-grained control of the feature. + +#### Technical Details + +Some points, in greater detail: + +##### Traffic Access + +1. Ideally these services should not be exposed externally to Internet traffic: only internally to our existing Rails and Sidekiq workloads should be routed. + 1. For services intended to run at `M2`: "SAAS-hosted for use with self-managed instance and instance-hosted", we would expect to migrate the service to a public endpoint once sufficient security review has been performed. + +##### Platform Requirements + +In order to quickly deploy and manage experiments, an minimally viable platform will need to be provided to stage-group teams. The technical implementation details of this platform are out of scope for this blueprint and will require their own blueprint (to follow). + +However, Service-Integration will establish certain necessary and optional requirements that the platform will need to satisfy. + +###### Ease of Use, Ownership Requirements + +1. <a name="R100">`R100`</a>: Required: the platform should be easy to use: imagine Heroku with [GitLab Production Readiness-approved](https://about.gitlab.com/handbook/engineering/infrastructure/production/readiness/) defaults. +1. <a name="R110">`R110`</a>: Required: with the exception of an Infrastructure-led onboarding process, services are owned, deployed and managed by stage-group teams. In other words,services follow a "You Build It, You Run It" model of ownership. +1. <a name="R120">`R120`</a>: Required: programming-language agnostic: no requirements for services. Services should be packaged as container images. +1. <a name="R130">`R130`</a>: Recommended: Each service should be evaluated against the GitLab.com [Service Maturity Model](https://about.gitlab.com/handbook/engineering/infrastructure/service-maturity-model/). +1. <a name="R140">`R140`</a>: Recommended: services using the platform have expedited production-readiness processes. + 1. Production-readiness requirements graded by service maturity: low-traffic, low-maturity experimental services will have lower requirement thresholds than more mature services. + 1. By default, the platform should provide services with defaults that would pass production-readiness review for the lowest service maturity-level. + 1. At introduction, lowest maturity services can be deployed without production readiness, provided the meet certain automatically validated requirements. This removes Infrastructure gate-keeping from being a blocker to experimental service delivery. + +###### Observability Requirements + +1. <a name="R200">`R200`</a>: Required: the platform must provide SLIs for services out-of-the-box. + 1. While it is recommended that services expose internal metrics, it is not mandatory. The platform will provide monitoring from the load-balancer. This is to speed up deployment by removing barriers to experimentation. + 1. For services that provide internal metrics scrape endpoints, the platform must be configurable to collect these. + 1. The platform must provide generic load-balancer level SLIs for all services. Service owners must be able to select from constructing SLIs from internal application metrics, the platform-provided external SLIs, or a combination of both. +1. <a name="R210">`R210`</a>: Required: Observability dashboards, rules, alerts (with per-term routing) must be generated from a manifest. +1. <a name="R220">`R220`</a>:Required: standardized logging infrastructure. + 1. Mandate that all logging emitted from services must be Structured JSON. Text logs are permitted but not recommended. + 1. See [Common Service Libraries](#common-service-libraries) for more details of building common SDKs for observability. + +###### Deployment Requirements + +1. <a name="R300">`R300`</a>: Required: No secrets stored in CI/CD. + 1. Authentication with Cloud Provider Resources should be exclusively via OIDC, managed as part of the platform. + 1. Secrets should be stored in the Infrastructure-provided Hashicorp Vault for the environment and passed to applications through files or environment variables. + 1. Generation and management of service account tokens should be done declaratively, without manual interaction. +1. <a name="R310">`R310`</a>: Required: multiple environment should be supported, eg Staging and Production. +1. <a name="R320">`R320`</a>: Required: the platform should be cost-effective. Kubernetes clusters should support multiple services and teams. +1. <a name="R330">`R330`</a>: Recommended: gradual rollouts, rollbacks, blue-green deployments. +1. <a name="R340">`R340`</a>: Required: services should be isolated from one another. +1. <a name="R350">`R350`</a>: Recommended: services should have the ability to specify node characteristic requirements (eg, GPU). +1. <a name="R360">`R360`</a>: Required: Developers should not need knowledge of Helm, Kubernetes, Prometheus in order to deploy. All required values are configured and validated in project-hosted manifest before generating Kubernetes manifests, Prometheus rules, etc. +1. <a name="R370">`R370`</a>: Initially services should be synchronous only - using REST or GRPC requests. + 1. This does not however preclude long-running HTTP(s) requests, for example long-polling or Websocket requests. +1. <a name="R390">`R390`</a>: Each service hosted in its own GitLab repository with deployment manifest stored in the repository. + 1. Continuous deployments that are initiated from the CI pipeline of the corresponding GitLab repository. + +##### Security Requirements + +1. <a name="R400">`R400`</a>: stateful services deployed on the platform that utilize their own stateful storage (for example, custom deployed Postgres instance), must not store application security tokens, cloud-provider service keys or other long-lived security tokens in their stateful stores. +1. <a name="R410">`R410`</a>: long-lived shared secrets are discouraged, and should be referenced in the service manifest as such, to allow for accounting and monitoring. +1. <a name="R420">`R420`</a>: services using long-lived shared secrets should ensure that secret rotation can take place without downtime. + 1. During a rotation, old and new generations of secrets should pass authentication, allowing gradual roll-out of new secrets. + +##### Common Service Libraries + +1. <a name="R500">`R500`</a>: Experimental services would be strongly encouraged to adopt and use [LabKit](https://gitlab.com/gitlab-org/labkit) (for Go services), or [LabKit-Ruby](https://gitlab.com/gitlab-org/ruby/gems/labkit-ruby) for observability, context, correlation, FIPs verification, etc. + 1. At present, there is no LabKit-Python library, but some experiments will run in Python, so building a library to providing observability, context, correlation services in Python will be required. diff --git a/doc/ci/yaml/includes.md b/doc/ci/yaml/includes.md index cd2cb3c96e1..35b302d0373 100644 --- a/doc/ci/yaml/includes.md +++ b/doc/ci/yaml/includes.md @@ -603,3 +603,6 @@ To help reduce the risk of this happening, edit the pipeline configuration file with the [pipeline editor](../pipeline_editor/index.md), which validates if the limit is reached. You can remove one included file at a time to try to narrow down which configuration file is the source of the loop or excessive included files. + +In [GitLab 16.0 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/207270) self-managed users can +change the [maximum includes](../../user/admin_area/settings/continuous_integration.md#maximum-includes) value. diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index 5090e016cb0..39f979d98d5 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -136,16 +136,6 @@ The `include` files are: - Always evaluated first and then merged with the content of the `.gitlab-ci.yml` file, regardless of the position of the `include` keyword. -You can have up to 150 includes per pipeline, including [nested](includes.md#use-nested-includes) includes: - -- In [GitLab 15.10 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/367150) you can have up to 150 includes. - In nested includes, the same file can be included multiple times, but duplicated includes - count towards the limit. -- From [GitLab 14.9 to GitLab 15.9](https://gitlab.com/gitlab-org/gitlab/-/issues/28987), you can have up to 100 includes. - The same file can be included multiple times in nested includes, but duplicates are ignored. -- In GitLab 14.9 and earlier you can have up to 100 includes, but the same file can not - be included multiple times. - The time limit to resolve all files is 30 seconds. **Keyword type**: Global keyword. @@ -171,6 +161,16 @@ The time limit to resolve all files is 30 seconds. do not affect job reruns. - Pipeline, the `include` files are fetched again. If they changed after the last pipeline run, the new pipeline uses the changed configuration. +- You can have up to 150 includes per pipeline by default, including [nested](includes.md#use-nested-includes). Additionally: + - In [GitLab 16.0 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/207270) self-managed users can + change the [maximum includes](../../user/admin_area/settings/continuous_integration.md#maximum-includes) value. + - In [GitLab 15.10 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/367150) you can have up to 150 includes. + In nested includes, the same file can be included multiple times, but duplicated includes + count towards the limit. + - From [GitLab 14.9 to GitLab 15.9](https://gitlab.com/gitlab-org/gitlab/-/issues/28987), you can have up to 100 includes. + The same file can be included multiple times in nested includes, but duplicates are ignored. + - In GitLab 14.9 and earlier you can have up to 100 includes, but the same file can not + be included multiple times. **Related topics**: diff --git a/doc/development/github_importer.md b/doc/development/github_importer.md index 86142efa679..bca1fc276d5 100644 --- a/doc/development/github_importer.md +++ b/doc/development/github_importer.md @@ -146,10 +146,10 @@ comments. This worker imports note attachments that are linked inside Markdown. For each entity with Markdown text in the project, we schedule a job of: -- `Gitlab::GithubImport::ImportReleaseAttachmentsWorker` for every release. -- `Gitlab::GithubImport::ImportNoteAttachmentsWorker` for every note. -- `Gitlab::GithubImport::ImportIssueAttachmentsWorker` for every issue. -- `Gitlab::GithubImport::ImportMergeRequestAttachmentsWorker` for every merge request. +- `Gitlab::GithubImport::Attachments::ImportReleaseWorker` for every release. +- `Gitlab::GithubImport::Attachments::ImportNoteWorker` for every note. +- `Gitlab::GithubImport::Attachments::ImportIssueWorker` for every issue. +- `Gitlab::GithubImport::Attachments::ImportMergeRequestWorker` for every merge request. Each job: diff --git a/doc/development/integrations/jira_connect.md b/doc/development/integrations/jira_connect.md index 98a497518cb..b265f84bc17 100644 --- a/doc/development/integrations/jira_connect.md +++ b/doc/development/integrations/jira_connect.md @@ -74,7 +74,8 @@ To avoid external dependencies like Gitpod and a Jira Cloud instance, use the [J ## Test the GitLab OAuth authentication flow -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81126) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `jira_connect_oauth`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81126) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `jira_connect_oauth`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117648) in GitLab 16.0. Feature flag `jira_connect_oauth` removed. GitLab for Jira users can authenticate with GitLab using GitLab OAuth. @@ -83,18 +84,7 @@ This feature is not ready for production use. The feature flag should only be en The following steps describe setting up an environment to test the GitLab OAuth flow: -1. Start a Gitpod session and open the rails console. - - ```shell - bundle exec rails console - ``` - -1. Enable the feature flag. - - ```shell - Feature.enable(:jira_connect_oauth) - ``` - +1. Start a Gitpod session. 1. On your GitLab instance, go to **Admin > Applications**. 1. Create a new application with the following settings: - Name: `Jira Connect` diff --git a/doc/development/pipelines/index.md b/doc/development/pipelines/index.md index ea02e920c6c..b3d789346cc 100644 --- a/doc/development/pipelines/index.md +++ b/doc/development/pipelines/index.md @@ -302,6 +302,7 @@ runs the GitLab test suite "as if JiHu", meaning as if the pipeline would run in the context of [GitLab JH](../jh_features_review.md). These jobs are only created in the following cases: +- when changes are made to feature flags - when the `pipeline:run-as-if-jh` label is set on the merge request This pipeline runs under the context of a generated branch in the diff --git a/doc/development/testing_guide/frontend_testing.md b/doc/development/testing_guide/frontend_testing.md index a3b580a43fa..ba965ab204a 100644 --- a/doc/development/testing_guide/frontend_testing.md +++ b/doc/development/testing_guide/frontend_testing.md @@ -894,6 +894,26 @@ You can generate fixtures by running: You can find generated fixtures are in `tmp/tests/frontend/fixtures-ee`. +### Download fixtures + +We generate fixtures in GitLab CI, and store them in the package registry. + +The `scripts/frontend/download_fixtures.sh` script is meant to download and extract those fixtures for local use: + +```shell +# Checks if a frontend fixture package exists in the gitlab-org/gitlab +# package registry by looking at the commits on a local branch. +# +# The package is downloaded and extracted if it exists +$ scripts/frontend/download_fixtures.sh + +# Same as above, but only looks at the last 10 commits of the currently checked-out branch +$ scripts/frontend/download_fixtures.sh --max-commits=10 + +# Looks at the commits on the local master branch instead of the currently checked-out branch +$ scripts/frontend/download_fixtures.sh --branch master +``` + #### Creating new fixtures For each fixture, you can find the content of the `response` variable in the output file. diff --git a/doc/integration/jira/configure.md b/doc/integration/jira/configure.md index f1f7a34ac94..7503ae993dc 100644 --- a/doc/integration/jira/configure.md +++ b/doc/integration/jira/configure.md @@ -4,22 +4,20 @@ group: Integrations info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Jira integration **(FREE)** +# Jira issue integration **(FREE)** -The Jira integration connects one or more GitLab projects to a Jira instance. You can host the Jira instance yourself or in [Atlassian Cloud](https://www.atlassian.com/migration/assess/why-cloud). The supported Jira versions are `6.x`, `7.x`, `8.x`, and `9.x`. +The Jira issue integration connects one or more GitLab projects to a Jira instance. You can host the Jira instance yourself or in [Jira Cloud](https://www.atlassian.com/migration/assess/why-cloud). The supported Jira versions are `6.x`, `7.x`, `8.x`, and `9.x`. ## Configure the integration Prerequisites: -- Ensure your GitLab installation does not use a [relative URL](https://docs.gitlab.com/omnibus/settings/configuration.html#configure-a-relative-url-for-gitlab). -- For **Jira Server**, ensure you have a Jira username and password. - See [authentication in Jira](index.md#authentication-in-jira). -- For **Jira Cloud**, ensure you have an API token - and the email address you used to create the token. - See [authentication in Jira](index.md#authentication-in-jira). +- Your GitLab installation must not use a [relative URL](https://docs.gitlab.com/omnibus/settings/configuration.html#configure-a-relative-url-for-gitlab). +- **For Jira Server**, you must have a Jira username and password. +- **For Jira Cloud**, you must have a [Jira Cloud API token](#create-a-jira-cloud-api-token) and + the email address you used to create the token. -You can enable the Jira integration by configuring your project settings in GitLab. +You can enable the Jira issue integration by configuring your project settings in GitLab. You can also configure these settings at the: - [Group level](../../user/admin_area/settings/project_integration_management.md#manage-group-level-default-settings-for-a-project-integration) @@ -66,7 +64,7 @@ displays a Jira link that opens the Jira project. ## Create a Jira Cloud API token -To [integrate with Jira](index.md) in Atlassian Cloud, you must create an API token. +To configure the integration with Jira Cloud, you must create a Jira Cloud API token. To create a Jira Cloud API token: @@ -81,11 +79,6 @@ To create a Jira Cloud API token: To copy the API token, select **Copy** and paste the token somewhere safe. -To configure GitLab, you need: - -- The newly created token -- The email address you used when you created the token - ## Migrate from Jira Server to Jira Cloud in GitLab To migrate from Jira Server to Jira Cloud in GitLab and maintain your Jira integration: @@ -95,7 +88,7 @@ To migrate from Jira Server to Jira Cloud in GitLab and maintain your Jira integ 1. Select **Jira**. 1. In **Web URL**, enter the new Jira site URL (for example, `https://myjirasite.atlassian.net`). 1. In **Username or Email**, enter the username or email registered on your Jira profile. -1. [Create an API token](#create-a-jira-cloud-api-token), and copy that value. +1. [Create a Jira Cloud API token](#create-a-jira-cloud-api-token), and copy the token value. 1. In **Password or API token**, paste the API token value. 1. Optional. Select **Test settings** to check if the connection is working. 1. Select **Save changes**. diff --git a/doc/integration/jira/connect-app.md b/doc/integration/jira/connect-app.md index 7ef6a542993..617de05d480 100644 --- a/doc/integration/jira/connect-app.md +++ b/doc/integration/jira/connect-app.md @@ -79,7 +79,6 @@ To create an OAuth application: 1. Expand **GitLab for Jira App**. 1. Paste the **Application ID** value into **Jira Connect Application ID**. 1. Select **Save changes**. -1. Optional. Enable the `jira_connect_oauth` [feature flag](../../administration/feature_flags.md) to avoid [authentication problems in some browsers](#browser-displays-a-sign-in-message-when-already-signed-in). ## Connect the GitLab for Jira Cloud app for self-managed instances **(FREE SELF)** @@ -252,7 +251,7 @@ You need to sign in or sign up before continuing. The GitLab for Jira Cloud app uses an iframe to add namespaces on the settings page. Some browsers block cross-site cookies, which can lead to this issue. -To resolve this issue, set up [OAuth authentication](#set-up-oauth-authentication-for-self-managed-instances) and enable the `jira_connect_oauth` [feature flag](../../administration/feature_flags.md). +To resolve this issue, set up [OAuth authentication](#set-up-oauth-authentication-for-self-managed-instances). ### Manual installation fails diff --git a/doc/integration/jira/index.md b/doc/integration/jira/index.md index 5b60994e94c..44ac4ef2d07 100644 --- a/doc/integration/jira/index.md +++ b/doc/integration/jira/index.md @@ -6,38 +6,34 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Jira **(FREE)** -If your organization uses [Jira](https://www.atlassian.com/software/jira) issues, -you can [migrate your issues from Jira](../../user/project/import/jira.md) and work -exclusively in GitLab. However, if you'd like to continue to use Jira, you can -integrate it with GitLab. GitLab offers two types of Jira integrations, and you -can use one or both depending on the capabilities you need. +If your organization uses [Jira](https://www.atlassian.com/software/jira), +you can [migrate your issues from Jira to GitLab](../../user/project/import/jira.md). +If you want to continue to use Jira, you can integrate Jira with GitLab instead. -## Compare integrations +## Jira integrations -After you set up one or both of these integrations, you can cross-reference activity -in your GitLab project with any of your projects in Jira. +GitLab offers two types of Jira integrations. You can +use one or both depending on the capabilities you need. -### Jira integration +### Jira issue integration -The Jira integration connects one or more GitLab projects to a Jira instance. You can host -the Jira instance yourself or in [Atlassian Cloud](https://www.atlassian.com/migration/assess/why-cloud). -The supported Jira versions are `6.x`, `7.x`, `8.x`, and `9.x`. +You can use the [Jira issue integration](configure.md) developed by GitLab with Jira Cloud, Jira Data Center, or Jira Server. With this integration, you can: -For more information, see [Jira integration](configure.md). +- View and search Jira issues directly in GitLab. +- Refer to Jira issues by ID in GitLab commits and merge requests. +- Create Jira issues for vulnerabilities. ### Jira development panel -The [Jira development panel](development_panel.md) connects all GitLab projects in a group or personal namespace. -When you configure the Jira development panel, you can -[view GitLab activity for an issue](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/) -including related branches, commits, and merge requests. +You can use the [Jira development panel](development_panel.md) to [view GitLab activity for an issue](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/) +including related branches, commits, and merge requests. To configure the Jira development panel: -To set up the Jira development panel, use the GitLab for Jira Cloud app -or the Jira DVCS connector. For more information, see [Configure the panel](development_panel.md#configure-the-panel). +- **For Jira Cloud**, use the [GitLab for Jira Cloud app](connect-app.md) developed by GitLab. +- **For Jira Data Center or Jira Server**, use the [Jira DVCS connector](dvcs/index.md) developed by Atlassian. -### Direct feature comparison +## Jira integration capabilities -| Capability | Jira integration | Jira development panel | +| Capability | Jira issue integration | Jira development panel | |-|-|-| | Mention a Jira issue ID in a GitLab commit or merge request, and a link to the Jira issue is created. | Yes. | No. | | Mention a Jira issue ID in GitLab and the Jira issue shows the GitLab issue or merge request. | Yes. A Jira comment with the GitLab issue or MR title links to GitLab. The first mention is also added to the Jira issue under **Web links**. | Yes, in the issue's [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/). | @@ -50,25 +46,13 @@ or the Jira DVCS connector. For more information, see [Configure the panel](deve | Create a GitLab branch from a Jira issue. | No. | Yes, in the issue's [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/). | | Mention a Jira issue ID in a GitLab merge request, and deployments are synced. | No. | Yes, in the issue's [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/). | -## Authentication in Jira - -The authentication method in Jira depends on whether you host Jira on your own server or on -[Atlassian cloud](https://www.atlassian.com/migration/assess/why-cloud): - -- **Jira Server** supports basic authentication. When connecting, a **username and password** are - required. Connecting to Jira Server using the Central Authentication Service (CAS) is not possible. For more information, read - how to [set up a user in Jira Server](jira_server_configuration.md). -- **Jira on Atlassian cloud** supports authentication through an API token. When connecting to Jira on - Atlassian cloud, an email and API token are required. For more information, see - [Create a Jira Cloud API token](configure.md#create-a-jira-cloud-api-token). - ## Privacy considerations All Jira integrations share data with Jira to make it visible outside of GitLab. If you integrate a private GitLab project with Jira, the private data is shared with users who have access to your Jira project. -The [Jira integration](#jira-integration) posts GitLab data in the form of comments in Jira issues. +The [Jira issue integration](configure.md) posts GitLab data in the form of comments in Jira issues. The GitLab for Jira Cloud app and Jira DVCS connector share this data through the [Jira development panel](development_panel.md). This method provides more fine-grained access control because access can be restricted to certain user groups or roles. diff --git a/doc/integration/jira/issues.md b/doc/integration/jira/issues.md index e7802a5fde2..fcf0bf791d4 100644 --- a/doc/integration/jira/issues.md +++ b/doc/integration/jira/issues.md @@ -6,12 +6,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Jira issue management **(FREE)** -To integrate issue management with Jira, [configure Jira](index.md#jira-integration) -and [enable the integration](configure.md) in GitLab. -After you configure and enable the integration, you can reference and close Jira -issues by mentioning the Jira ID in GitLab commits and merge requests. - -The Jira integration requires to you format any Jira issue IDs in uppercase. +You can [manage Jira issues directly in GitLab](configure.md). +You can then refer to Jira issues by ID in GitLab commits and merge requests. +The Jira issue IDs must be in uppercase. ## Reference Jira issues diff --git a/doc/user/admin_area/settings/continuous_integration.md b/doc/user/admin_area/settings/continuous_integration.md index a0a0dc65099..d6b1d243d82 100644 --- a/doc/user/admin_area/settings/continuous_integration.md +++ b/doc/user/admin_area/settings/continuous_integration.md @@ -194,6 +194,16 @@ To set all new [CI/CD variables](../../../ci/variables/index.md) as 1. On the left sidebar, select **Settings > CI/CD**. 1. Select **Protect CI/CD variables by default**. +## Maximum includes + +The maximum number of [includes](../../../ci/yaml/includes.md) per pipeline can be set at the instance level. +The default is `150`. + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > CI/CD**. +1. Change the value of **Maximum includes**. +1. Select **Save changes** for the changes to take effect. + ## Default CI/CD configuration file > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18073) in GitLab 12.5. diff --git a/doc/user/group/access_and_permissions.md b/doc/user/group/access_and_permissions.md index ad27255f42e..114e5bd7bbf 100644 --- a/doc/user/group/access_and_permissions.md +++ b/doc/user/group/access_and_permissions.md @@ -64,7 +64,7 @@ To change the permitted Git access protocols for a group: To ensure only people from your organization can access particular resources, you can restrict access to groups by IP address. This group-level setting applies to: -- The GitLab UI, including subgroups, projects, and issues. +- The GitLab UI, including subgroups, projects, and issues. It does not apply to GitLab Pages. - [In GitLab 12.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/12874), the API. - In self-managed installations of GitLab 15.1 and later, you can also configure [globally-allowed IP address ranges](../admin_area/settings/visibility_and_access_controls.md#configure-globally-allowed-ip-address-ranges) diff --git a/doc/user/product_analytics/index.md b/doc/user/product_analytics/index.md index 4c5733ed5ec..dd860fb0c82 100644 --- a/doc/user/product_analytics/index.md +++ b/doc/user/product_analytics/index.md @@ -100,7 +100,7 @@ On GitLab.com, this feature is not available. This feature is not ready for production use. Each project can have an unlimited number of dashboards. -These dashboards are defined using the GitLab YAML schema, and stored in the `.gitlab/product_analytics/dashboards/` directory of a project repository. +These dashboards are defined using the GitLab YAML schema, and stored in the `.gitlab/analytics/dashboards/` directory of a project repository. The name of the file is the name of the dashboard. Each dashboard can contain one or more visualizations (charts), which are shared across dashboards. @@ -126,13 +126,13 @@ To view a list of product analytics dashboards for a project: To define a dashboard: -1. In `.gitlab/product_analytics/dashboards/`, create a directory named like the dashboard. +1. In `.gitlab/analytics/dashboards/`, create a directory named like the dashboard. Each dashboard should have its own directory. 1. In the new directory, create a `.yaml` file with the same name as the directory. This file contains the dashboard definition. It must conform to the JSON schema defined in `ee/app/validators/json_schemas/product_analytics_dashboard.json`. -1. In the `.gitlab/product_analytics/dashboards/visualizations/` directory, create a `.yaml` file. +1. In the `.gitlab/analytics/dashboards/visualizations/` directory, create a `.yaml` file. This file defines the visualization type for the dashboard. It must conform to the schema in `ee/app/validators/json_schemas/product_analytics_visualization.json`. @@ -141,7 +141,7 @@ For example, if you want to create three dashboards (Conversion funnels, Demogra and one visualization (line chart) that applies to all dashboards, the file structure would be: ```plaintext -.gitlab/product_analytics/dashboards +.gitlab/analytics/dashboards ├── conversion_funnels │ └── conversion_funnels.yaml ├── demographic_breakdown @@ -152,13 +152,38 @@ and one visualization (line chart) that applies to all dashboards, the file stru │ └── example_line_chart.yaml ``` +### Define a chart visualization + +You can define different charts, and add visualization options to some of them: + +- Line chart, with the options listed in the [ECharts documentation](https://echarts.apache.org/en/option.html). +- Column chart, with the options listed in the [ECharts documentation](https://echarts.apache.org/en/option.html). +- Data table, without options. +- Single stat, with the only option to set `decimalPlaces` (number, default value is 0). + +To define a chart for your dashboards: + +1. In the `.gitlab/product_analytics/dashboards/visualizations/` directory, create a `.yaml` file. +The filename should be descriptive of the visualization it defines. +1. In the `.yaml` file, define the visualization options, according to the schema in +`ee/app/validators/json_schemas/analytics_visualization.json`. + +For [example](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/assets/javascripts/analytics/analytics_dashboards/gl_dashboards/product_analytics/visualizations/events_over_time.json), to create a line chart that illustrates event count over time, in the `visualizations` folder +create a `line_chart.yaml` file with the following required fields: + +- version +- title +- type +- data +- options + ## Funnel analysis Use funnel analysis to understand the flow of users through your application, and where users drop out of a predefined flow (for example, a checkout process or ticket purchase). Each product can also define an unlimited number of funnels. -Like dashboards, funnels are defined using the GitLab YAML schema, and stored in the `.gitlab/product_analytics/funnels/` directory of a project repository. +Like dashboards, funnels are defined using the GitLab YAML schema, and stored in the `.gitlab/analytics/funnels/` directory of a project repository. Funnel definitions must include the keys `name` and `seconds_to_convert`, and an array of `steps`. diff --git a/jest_resolver.js b/jest_resolver.js index 6cbc1b9f18f..66434ca7a6c 100644 --- a/jest_resolver.js +++ b/jest_resolver.js @@ -9,7 +9,7 @@ module.exports = (request, options) => { console.error( '\x1b[1m\x1b[41m\x1b[30m %s \x1b[0m %s', '!', - `Fixture file ${request} does not exist. Did you run bin/rake frontend:fixtures?`, + `Fixture file ${request} does not exist. Did you run bin/rake frontend:fixtures? You can also download fixtures from the gitlab-org/gitlab package registry. See the Fixtures docs for more info.`, ); } throw e; diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 9562af225a7..f3204104b9f 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -196,6 +196,7 @@ module API optional :jira_connect_proxy_url, type: String, desc: "URL of the GitLab instance that should be used as a proxy for the GitLab for Jira Cloud app" optional :bulk_import_enabled, type: Boolean, desc: 'Enable migrating GitLab groups and projects by direct transfer' optional :allow_runner_registration_token, type: Boolean, desc: 'Allow registering runners using a registration token' + optional :ci_max_includes, type: Integer, desc: 'Maximum number of includes per pipeline' Gitlab::SSHPublicKey.supported_types.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 156109a084d..b8e012ec851 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -9,8 +9,7 @@ module Gitlab TimeoutError = Class.new(StandardError) - MAX_INCLUDES = 150 - TEMP_MAX_INCLUDES = 100 # For logging; to be removed in https://gitlab.com/gitlab-org/gitlab/-/issues/367150 + TEMP_MAX_INCLUDES = 100 # For logging; to be removed in https://gitlab.com/gitlab-org/gitlab/-/issues/396776 include ::Gitlab::Utils::StrongMemoize @@ -32,7 +31,7 @@ module Gitlab @expandset = [] @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) - @max_includes = MAX_INCLUDES + @max_includes = Gitlab::CurrentSettings.current_application_settings.ci_max_includes yield self if block_given? end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index bef4b147359..fee01b8a22a 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -7,14 +7,14 @@ module Gitlab module Validators class SchemaValidator SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], - container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], - coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], - dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], - api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], - dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], - sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], - secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4] + cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], + container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], + coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], + dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], + api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], + dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], + sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], + secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6] }.freeze VERSIONS_TO_REMOVE_IN_16_0 = %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3].freeze diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..91414255211 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/cluster-image-scanning-report-format.json @@ -0,0 +1,1035 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.6" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/container-scanning-report-format.json new file mode 100644 index 00000000000..ecd92ed2ff1 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/container-scanning-report-format.json @@ -0,0 +1,967 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.6" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..11a1375710b --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/coverage-fuzzing-report-format.json @@ -0,0 +1,925 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.6" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dast-report-format.json new file mode 100644 index 00000000000..1351cb261e0 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dast-report-format.json @@ -0,0 +1,1330 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.6" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dependency-scanning-report-format.json new file mode 100644 index 00000000000..e4b02362cb1 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dependency-scanning-report-format.json @@ -0,0 +1,1033 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.6" + }, + "type": "object", + "required": [ + "dependency_files", + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/sast-report-format.json new file mode 100644 index 00000000000..e4cb5fb2985 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/sast-report-format.json @@ -0,0 +1,920 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.6" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/secret-detection-report-format.json new file mode 100644 index 00000000000..5eb52b11efe --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/secret-detection-report-format.json @@ -0,0 +1,944 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.6" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb index 619f11ae890..98b44934d41 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb @@ -5,6 +5,7 @@ module Gitlab module LoadBalancing class SidekiqClientMiddleware include Gitlab::Utils::StrongMemoize + include WalTrackingSender def call(worker_class, job, _queue, _redis_pool) # Mailers can't be constantized @@ -30,15 +31,7 @@ module Gitlab end def set_data_consistency_locations!(job) - locations = {} - - ::Gitlab::Database::LoadBalancing.each_load_balancer do |lb| - if (location = wal_location_for(lb)) - locations[lb.name] = location - end - end - - job['wal_locations'] = locations + job['wal_locations'] = wal_locations_by_db_name job['wal_location_source'] = wal_location_source end @@ -50,17 +43,6 @@ module Gitlab end end - def wal_location_for(load_balancer) - # When only using the primary there's no need for any WAL queries. - return if load_balancer.primary_only? - - if uses_primary? - load_balancer.primary_write_location - else - load_balancer.host&.database_replica_location || load_balancer.primary_write_location - end - end - def uses_primary? ::Gitlab::Database::LoadBalancing::Session.current.use_primary? end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index 95e21c40795..dc8b44f8f2f 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -4,6 +4,8 @@ module Gitlab module Database module LoadBalancing class SidekiqServerMiddleware + include WalTrackingReceiver + JobReplicaNotUpToDate = Class.new(::Gitlab::SidekiqMiddleware::RetryError) REPLICA_WAIT_SLEEP_SECONDS = 0.5 @@ -92,19 +94,6 @@ module Gitlab # the `0` indicates that this is a first retry job['retry_count'].nil? end - - def databases_in_sync?(wal_locations) - ::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb| - if (location = wal_locations.with_indifferent_access[lb.name]) - lb.select_up_to_date_host(location) - else - # If there's no entry for a load balancer it means the Sidekiq - # job doesn't care for it. In this case we'll treat the load - # balancer as being in sync. - true - end - end - end end end end diff --git a/lib/gitlab/database/load_balancing/wal_tracking_receiver.rb b/lib/gitlab/database/load_balancing/wal_tracking_receiver.rb new file mode 100644 index 00000000000..ef9bee42277 --- /dev/null +++ b/lib/gitlab/database/load_balancing/wal_tracking_receiver.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + module WalTrackingReceiver + # NOTE: If there's no entry for a load balancer or no WAL locations were passed + # we assume the sender does not care about LB and we assume nodes are in-sync. + def databases_in_sync?(wal_locations) + return true unless wal_locations.present? + + ::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb| + if (location = wal_locations.with_indifferent_access[lb.name]) + lb.select_up_to_date_host(location) + else + true + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/wal_tracking_sender.rb b/lib/gitlab/database/load_balancing/wal_tracking_sender.rb new file mode 100644 index 00000000000..ed0af58d064 --- /dev/null +++ b/lib/gitlab/database/load_balancing/wal_tracking_sender.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + module WalTrackingSender + def wal_locations_by_db_name + {}.tap do |locations| + ::Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + if (location = wal_location_for(lb)) + locations[lb.name] = location + end + end + end + end + + def wal_location_for(load_balancer) + # When only using the primary there's no need for any WAL queries. + return if load_balancer.primary_only? + + if Session.current.use_primary? + load_balancer.primary_write_location + else + load_balancer.host&.database_replica_location || load_balancer.primary_write_location + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb b/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb index a74d8982d73..1a4dd936f21 100644 --- a/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb +++ b/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb @@ -5,15 +5,56 @@ module Gitlab module Subscriptions class ActionCableWithLoadBalancing < ::GraphQL::Subscriptions::ActionCableSubscriptions extend ::Gitlab::Utils::Override + include Gitlab::Database::LoadBalancing::WalTrackingReceiver - # When executing updates we are usually responding to a broadcast as a result of a DB update. - # We use the primary so that we are sure that we are returning the newly updated data. + def initialize(**options) + super(serializer: WalInjectingSerializer.new, **options) + end + + # We fall back to the primary in case no replica is sufficiently caught up. override :execute_update def execute_update(subscription_id, event, object) - ::Gitlab::Database::LoadBalancing::Session.current.use_primary! + ::Gitlab::Database::LoadBalancing::Session.current.use_primary! if use_primary? super end + + private + + def use_primary? + @serializer.wal_locations.blank? || + Feature.disabled?(:graphql_subs_lb) || + !databases_in_sync?(@serializer.wal_locations) + end + end + + class WalInjectingSerializer + include Gitlab::Database::LoadBalancing::WalTrackingSender + + DEFAULT_SERIALIZER = GraphQL::Subscriptions::Serialize + + attr_reader :wal_locations + + # rubocop: disable GitlabSecurity/PublicSend + def load(str) + return DEFAULT_SERIALIZER.load(str) unless Feature.enabled?(:graphql_subs_lb) + + value = Gitlab::Json.parse(str) + + @wal_locations = value['wal_locations'] + + DEFAULT_SERIALIZER.send(:load_value, value['payload']) + end + + def dump(obj) + return DEFAULT_SERIALIZER.dump(obj) unless Feature.enabled?(:graphql_subs_lb) + + Gitlab::Json.dump({ + 'wal_locations' => wal_locations_by_db_name, + 'payload' => DEFAULT_SERIALIZER.send(:dump_value, obj) + }) + end + # rubocop: enable GitlabSecurity/PublicSend end end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index c0c33ca73f8..2b1f777d27a 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -3081,6 +3081,9 @@ msgstr "" msgid "AdminSettings|Maximum duration of a session for Git operations when 2FA is enabled." msgstr "" +msgid "AdminSettings|Maximum includes" +msgstr "" + msgid "AdminSettings|Maximum number of DAG dependencies that a job can have" msgstr "" @@ -3234,6 +3237,9 @@ msgstr "" msgid "AdminSettings|The latest artifacts for all jobs in the most recent successful pipelines in each project are stored and do not expire." msgstr "" +msgid "AdminSettings|The maximum number of included files per pipeline." +msgstr "" + msgid "AdminSettings|The template for the required pipeline configuration can be one of the GitLab-provided templates, or a custom template added to an instance template repository. %{link_start}How do I create an instance template repository?%{link_end}" msgstr "" @@ -43695,9 +43701,27 @@ msgstr "" msgid "Take a look at the documentation to discover all of GitLab’s capabilities." msgstr "" +msgid "TanukiBot|Ask a question about GitLab" +msgstr "" + +msgid "TanukiBot|For example, %{linkStart}what is a fork?%{linkEnd}" +msgstr "" + +msgid "TanukiBot|Give feedback" +msgstr "" + +msgid "TanukiBot|Sources" +msgstr "" + msgid "TanukiBot|Tanuki Bot" msgstr "" +msgid "TanukiBot|There was an error communicating with Tanuki Bot. Please reach out to GitLab support for more assistance or try again later." +msgstr "" + +msgid "TanukiBot|What is a fork?" +msgstr "" + msgid "Target" msgstr "" diff --git a/package.json b/package.json index b884cd07239..a351a1122c8 100644 --- a/package.json +++ b/package.json @@ -56,8 +56,8 @@ "@gitlab/cluster-client": "^1.2.0", "@gitlab/favicon-overlay": "2.0.0", "@gitlab/fonts": "^1.2.0", - "@gitlab/svgs": "3.40.0", - "@gitlab/ui": "62.4.0", + "@gitlab/svgs": "3.42.0", + "@gitlab/ui": "62.5.0", "@gitlab/visual-review-tools": "1.7.3", "@gitlab/web-ide": "0.0.1-dev-20230425040132", "@mattiasbuelens/web-streams-adapter": "^0.1.0", diff --git a/scripts/frontend/download_fixtures.sh b/scripts/frontend/download_fixtures.sh new file mode 100755 index 00000000000..47a57401bb9 --- /dev/null +++ b/scripts/frontend/download_fixtures.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# +# Downloads the most recent frontend fixtures for the current commit, going up the commit parent +# chain up to max-commits commits (defaults to 50 commits). +# + +source scripts/packages/helpers.sh + +print_help() { + echo "Usage: scripts/frontend/download_fixtures.sh [--branch <branch-name>] [--max-commits <number>]" + echo + echo "Looks for a frontend fixture package in the package registry for commits on a local branch." + echo + echo "If --branch isn't specified, the script will use the current branch as a commit reference." + echo "If --max-commits isn't specified, the default is 50 commits." + + return +} + +branch="HEAD" +max_commits_count=50 + +while [ $# -gt 0 ]; do + case "$1" in + --branch) + shift + branch="$1" + ;; + --max-commits) + shift + max_commits_count="$1" + ;; + *) + print_help + exit + ;; + esac + shift +done + +for commit_sha in $(git rev-list ${branch} --max-count="${max_commits_count}"); do + API_PACKAGES_BASE_URL=https://gitlab.com/api/v4/projects/278964/packages/generic + FIXTURES_PACKAGE="fixtures-${commit_sha}.tar.gz" + FIXTURES_PACKAGE_URL="${API_PACKAGES_BASE_URL}/fixtures/${commit_sha}/${FIXTURES_PACKAGE}" + + echo "Looking for frontend fixtures for commit ${commit_sha}..." + + if ! archive_doesnt_exist "${FIXTURES_PACKAGE_URL}" > /dev/null 2>&1; then + echo "We have found frontend fixtures at ${FIXTURES_PACKAGE_URL}!" + + read_curl_package "${FIXTURES_PACKAGE_URL}" | extract_package + + break + fi +done diff --git a/scripts/gitlab_component_helpers.sh b/scripts/gitlab_component_helpers.sh index 3d5116d6cc2..2e816330eb6 100644 --- a/scripts/gitlab_component_helpers.sh +++ b/scripts/gitlab_component_helpers.sh @@ -2,6 +2,9 @@ set -euo pipefail +# Generic helper functions for archives/packages +source scripts/packages/helpers.sh + export CURL_TOKEN_HEADER="${CURL_TOKEN_HEADER:-"JOB-TOKEN"}" export GITLAB_COM_CANONICAL_PROJECT_ID="278964" # https://gitlab.com/gitlab-org/gitlab @@ -54,63 +57,6 @@ export GITLAB_ASSETS_PACKAGE_URL="${API_PACKAGES_BASE_URL}/assets/${NODE_ENV}-${ # Fixtures constants export FIXTURES_PATH="tmp/tests/frontend/**/*" -# Generic helper functions -function archive_doesnt_exist() { - local package_url="${1}" - - status=$(curl -I --silent --retry 3 --output /dev/null -w "%{http_code}" "${package_url}") - - if [[ "${status}" = "200" ]]; then - echoinfo "The archive was found. The server returned status ${status}." - return 1 - else - echoinfo "The archive was not found. The server returned status ${status}." - return 0 - fi -} - -function create_package() { - local archive_filename="${1}" - local paths_to_archive="${2}" - local tar_working_folder="${3:-.}" - - echoinfo "Running 'tar -czvf ${archive_filename} -C ${tar_working_folder} ${paths_to_archive}'" - tar -czf ${archive_filename} -C ${tar_working_folder} ${paths_to_archive} - du -h ${archive_filename} -} - -function upload_package() { - local archive_filename="${1}" - local package_url="${2}" - local token_header="${CURL_TOKEN_HEADER}" - local token="${CI_JOB_TOKEN}" - - if [[ "${UPLOAD_PACKAGE_FLAG}" = "false" ]]; then - echoerr "The archive ${archive_filename} isn't supposed to be uploaded for this instance (${CI_SERVER_HOST}) & project (${CI_PROJECT_PATH})!" - exit 1 - fi - - echoinfo "Uploading ${archive_filename} to ${package_url} ..." - curl --fail --silent --retry 3 --header "${token_header}: ${token}" --upload-file "${archive_filename}" "${package_url}" -} - -function read_curl_package() { - local package_url="${1}" - - echoinfo "Downloading from ${package_url} ..." - - curl --fail --silent --retry 3 "${package_url}" -} - -function extract_package() { - local tar_working_folder="${1:-.}" - mkdir -p "${tar_working_folder}" - - echoinfo "Extracting archive to ${tar_working_folder}" - - tar -xz -C ${tar_working_folder} < /dev/stdin -} - # Workhorse functions function gitlab_workhorse_archive_doesnt_exist() { archive_doesnt_exist "${GITLAB_WORKHORSE_PACKAGE_URL}" diff --git a/scripts/packages/helpers.sh b/scripts/packages/helpers.sh new file mode 100644 index 00000000000..2917338aeb8 --- /dev/null +++ b/scripts/packages/helpers.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +source scripts/utils.sh + +function archive_doesnt_exist() { + local package_url="${1}" + + status=$(curl -I --silent --retry 3 --output /dev/null -w "%{http_code}" "${package_url}") + + if [[ "${status}" = "200" ]]; then + echoinfo "The archive was found. The server returned status ${status}." + return 1 + else + echoinfo "The archive was not found. The server returned status ${status}." + return 0 + fi +} + +function create_package() { + local archive_filename="${1}" + local paths_to_archive="${2}" + local tar_working_folder="${3:-.}" + + echoinfo "Running 'tar -czvf ${archive_filename} -C ${tar_working_folder} ${paths_to_archive}'" + tar -czf ${archive_filename} -C ${tar_working_folder} ${paths_to_archive} + du -h ${archive_filename} +} + +function upload_package() { + local archive_filename="${1}" + local package_url="${2}" + local token_header="${CURL_TOKEN_HEADER}" + local token="${CI_JOB_TOKEN}" + + if [[ "${UPLOAD_PACKAGE_FLAG}" = "false" ]]; then + echoerr "The archive ${archive_filename} isn't supposed to be uploaded for this instance (${CI_SERVER_HOST}) & project (${CI_PROJECT_PATH})!" + exit 1 + fi + + echoinfo "Uploading ${archive_filename} to ${package_url} ..." + curl --fail --silent --retry 3 --header "${token_header}: ${token}" --upload-file "${archive_filename}" "${package_url}" +} + +function read_curl_package() { + local package_url="${1}" + + echoinfo "Downloading from ${package_url} ..." + + curl --fail --silent --retry 3 "${package_url}" +} + +function extract_package() { + local tar_working_folder="${1:-.}" + mkdir -p "${tar_working_folder}" + + echoinfo "Extracting archive to ${tar_working_folder}" + + tar -xz -C ${tar_working_folder} < /dev/stdin +} diff --git a/spec/controllers/admin/application_settings_controller_spec.rb b/spec/controllers/admin/application_settings_controller_spec.rb index 253f66579f3..4b6274c2d26 100644 --- a/spec/controllers/admin/application_settings_controller_spec.rb +++ b/spec/controllers/admin/application_settings_controller_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Admin::ApplicationSettingsController, :do_not_mock_admin_mode_setting do +RSpec.describe Admin::ApplicationSettingsController, :do_not_mock_admin_mode_setting, feature_category: :shared do include StubENV include UsageDataHelpers @@ -398,6 +398,17 @@ RSpec.describe Admin::ApplicationSettingsController, :do_not_mock_admin_mode_set expect(application_settings.reload.invitation_flow_enforcement).to eq(true) end end + + context 'maximum includes' do + let(:application_settings) { ApplicationSetting.current } + + it 'updates ci_max_includes setting' do + put :update, params: { application_setting: { ci_max_includes: 200 } } + + expect(response).to redirect_to(general_admin_application_settings_path) + expect(application_settings.reload.ci_max_includes).to eq(200) + end + end end describe 'PUT #reset_registration_token', feature_category: :user_management do diff --git a/spec/features/admin/admin_settings_spec.rb b/spec/features/admin/admin_settings_spec.rb index 524fcb01644..4dee97880a5 100644 --- a/spec/features/admin/admin_settings_spec.rb +++ b/spec/features/admin/admin_settings_spec.rb @@ -426,6 +426,7 @@ RSpec.describe 'Admin updates settings', feature_category: :shared do fill_in 'application_setting_auto_devops_domain', with: 'domain.com' uncheck 'Keep the latest artifacts for all jobs in the latest successful pipelines' uncheck 'Enable pipeline suggestion banner' + fill_in 'application_setting_ci_max_includes', with: 200 click_button 'Save changes' end @@ -433,6 +434,7 @@ RSpec.describe 'Admin updates settings', feature_category: :shared do expect(current_settings.auto_devops_domain).to eq('domain.com') expect(current_settings.keep_latest_artifact).to be false expect(current_settings.suggest_pipeline_enabled).to be false + expect(current_settings.ci_max_includes).to be 200 expect(page).to have_content "Application settings saved successfully" end diff --git a/spec/features/projects/settings/repository_settings_spec.rb b/spec/features/projects/settings/repository_settings_spec.rb index 5e45d1683e7..08abade7d18 100644 --- a/spec/features/projects/settings/repository_settings_spec.rb +++ b/spec/features/projects/settings/repository_settings_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe 'Projects > Settings > Repository settings', feature_category: :projects do + include Features::MirroringHelpers + let(:project) { create(:project_empty_repo) } let(:user) { create(:user) } let(:role) { :developer } @@ -171,10 +173,8 @@ RSpec.describe 'Projects > Settings > Repository settings', feature_category: :p end it 'creates a push mirror that mirrors all branches', :js do - expect(page).to have_css('.js-mirror-protected-hidden[value="0"]', visible: false) - - fill_in 'url', with: ssh_url - expect(page).to have_css(".js-mirror-url-hidden[value=\"#{ssh_url}\"]", visible: false) + wait_for_mirror_field_javascript('protected', '0') + fill_and_wait_for_mirror_url_javascript('url', ssh_url) select 'SSH public key', from: 'Authentication method' select_direction @@ -191,10 +191,8 @@ RSpec.describe 'Projects > Settings > Repository settings', feature_category: :p it 'creates a push mirror that only mirrors protected branches', :js do find('#only_protected_branches').click - expect(page).to have_css('.js-mirror-protected-hidden[value="1"]', visible: false) - - fill_in 'url', with: ssh_url - expect(page).to have_css(".js-mirror-url-hidden[value=\"#{ssh_url}\"]", visible: false) + wait_for_mirror_field_javascript('protected', '1') + fill_and_wait_for_mirror_url_javascript('url', ssh_url) select 'SSH public key', from: 'Authentication method' select_direction @@ -211,8 +209,7 @@ RSpec.describe 'Projects > Settings > Repository settings', feature_category: :p it 'creates a push mirror that keeps divergent refs', :js do select_direction - fill_in 'url', with: ssh_url - expect(page).to have_css(".js-mirror-url-hidden[value=\"#{ssh_url}\"]", visible: false) + fill_and_wait_for_mirror_url_javascript('url', ssh_url) fill_in 'Password', with: 'password' check 'Keep divergent refs' @@ -231,8 +228,7 @@ RSpec.describe 'Projects > Settings > Repository settings', feature_category: :p end it 'generates an SSH public key on submission', :js do - fill_in 'url', with: ssh_url - expect(page).to have_css(".js-mirror-url-hidden[value=\"#{ssh_url}\"]", visible: false) + fill_and_wait_for_mirror_url_javascript('url', ssh_url) select 'SSH public key', from: 'Authentication method' diff --git a/spec/frontend/__helpers__/fixtures.js b/spec/frontend/__helpers__/fixtures.js index a6f7b37161e..c66411979e9 100644 --- a/spec/frontend/__helpers__/fixtures.js +++ b/spec/frontend/__helpers__/fixtures.js @@ -12,7 +12,10 @@ export function getFixture(relativePath) { throw new ErrorWithStack( `Fixture file ${relativePath} does not exist. -Did you run bin/rake frontend:fixtures?`, +Did you run bin/rake frontend:fixtures? You can also download fixtures from the gitlab-org/gitlab package registry. + +See https://docs.gitlab.com/ee/development/testing_guide/frontend_testing.html#download-fixtures for more info. +`, getFixture, ); } diff --git a/spec/frontend_integration/README.md b/spec/frontend_integration/README.md index 377294fb19f..ee760113307 100644 --- a/spec/frontend_integration/README.md +++ b/spec/frontend_integration/README.md @@ -22,6 +22,8 @@ We can generate the necessary fixtures and GraphQL schema by running: bundle exec rake frontend:fixtures gitlab:graphql:schema:dump ``` +You can also download those fixtures from the package registry: see [download fixtures](https://docs.gitlab.com/ee/development/testing_guide/frontend_testing.html#download-fixtures) for more info. + Then we can use [Jest](https://jestjs.io/) to run the frontend integration tests: ```shell diff --git a/spec/graphql/types/release_asset_link_type_spec.rb b/spec/graphql/types/release_asset_link_type_spec.rb index 0c903b8d27a..57a42a3966f 100644 --- a/spec/graphql/types/release_asset_link_type_spec.rb +++ b/spec/graphql/types/release_asset_link_type_spec.rb @@ -7,7 +7,7 @@ RSpec.describe GitlabSchema.types['ReleaseAssetLink'] do it 'has the expected fields' do expected_fields = %w[ - id name url external link_type direct_asset_url direct_asset_path + id name url link_type direct_asset_url direct_asset_path ] expect(described_class).to include_graphql_fields(*expected_fields) diff --git a/spec/helpers/jira_connect_helper_spec.rb b/spec/helpers/jira_connect_helper_spec.rb index 4f56bb7467f..454e4bf3fda 100644 --- a/spec/helpers/jira_connect_helper_spec.rb +++ b/spec/helpers/jira_connect_helper_spec.rb @@ -69,16 +69,6 @@ RSpec.describe JiraConnectHelper, feature_category: :integrations do ) end - context 'jira_connect_oauth feature is disabled' do - before do - stub_feature_flags(jira_connect_oauth: false) - end - - it 'does not assign oauth_metadata' do - expect(oauth_metadata).to be_nil - end - end - context 'with self-managed instance' do let_it_be(:installation) { create(:jira_connect_installation, instance_url: 'https://gitlab.example.com') } diff --git a/spec/lib/gitlab/ci/config/external/context_spec.rb b/spec/lib/gitlab/ci/config/external/context_spec.rb index fc68d3d62a2..d917924f257 100644 --- a/spec/lib/gitlab/ci/config/external/context_spec.rb +++ b/spec/lib/gitlab/ci/config/external/context_spec.rb @@ -24,7 +24,6 @@ RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipelin context 'with values' do it { is_expected.to have_attributes(**attributes) } it { expect(subject.expandset).to eq([]) } - it { expect(subject.max_includes).to eq(Gitlab::Ci::Config::External::Context::MAX_INCLUDES) } it { expect(subject.execution_deadline).to eq(0) } it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) } @@ -37,12 +36,27 @@ RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipelin it { is_expected.to have_attributes(**attributes) } it { expect(subject.expandset).to eq([]) } - it { expect(subject.max_includes).to eq(Gitlab::Ci::Config::External::Context::MAX_INCLUDES) } it { expect(subject.execution_deadline).to eq(0) } it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) } it { expect(subject.pipeline_config).to be_nil } end + + describe 'max_includes' do + it 'returns the default value of application setting `ci_max_includes`' do + expect(subject.max_includes).to eq(150) + end + + context 'when application setting `ci_max_includes` is changed' do + before do + stub_application_setting(ci_max_includes: 200) + end + + it 'returns the new value of application setting `ci_max_includes`' do + expect(subject.max_includes).to eq(200) + end + end + end end describe '#set_deadline' do diff --git a/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb b/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb index 83fc614bde3..2bacac06423 100644 --- a/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb @@ -64,47 +64,31 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware, feature shared_examples_for 'mark data consistency location' do |data_consistency| include_context 'data consistency worker class', data_consistency, :load_balancing_for_test_data_consistency_worker - - let(:location) { '0/D525E3A8' } + include_context 'when tracking WAL location reference' context 'when write was not performed' do before do - allow(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary?).and_return(false) + stub_no_writes_performed! end context 'when replica hosts are available' do it 'passes database_replica_location' do - expected_location = {} - - Gitlab::Database::LoadBalancing.each_load_balancer do |lb| - expect(lb.host) - .to receive(:database_replica_location) - .and_return(location) - - expected_location[lb.name] = location - end + expected_locations = expect_tracked_locations_when_replicas_available run_middleware - expect(job['wal_locations']).to eq(expected_location) + expect(job['wal_locations']).to eq(expected_locations) expect(job['wal_location_source']).to eq(:replica) end end context 'when no replica hosts are available' do it 'passes primary_write_location' do - expected_location = {} - - Gitlab::Database::LoadBalancing.each_load_balancer do |lb| - expect(lb).to receive(:host).at_least(:once).and_return(nil) - expect(lb).to receive(:primary_write_location).and_return(location) - - expected_location[lb.name] = location - end + expected_locations = expect_tracked_locations_when_no_replicas_available run_middleware - expect(job['wal_locations']).to eq(expected_location) + expect(job['wal_locations']).to eq(expected_locations) expect(job['wal_location_source']).to eq(:replica) end end @@ -114,23 +98,15 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware, feature context 'when write was performed' do before do - allow(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary?).and_return(true) + stub_write_performed! end it 'passes primary write location', :aggregate_failures do - expected_location = {} - - Gitlab::Database::LoadBalancing.each_load_balancer do |lb| - expect(lb) - .to receive(:primary_write_location) - .and_return(location) - - expected_location[lb.name] = location - end + expected_locations = expect_tracked_locations_from_primary_only run_middleware - expect(job['wal_locations']).to eq(expected_location) + expect(job['wal_locations']).to eq(expected_locations) expect(job['wal_location_source']).to eq(:primary) end diff --git a/spec/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing_spec.rb b/spec/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing_spec.rb index a9bf3f1dca9..1ab1a8ec2f5 100644 --- a/spec/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing_spec.rb +++ b/spec/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing_spec.rb @@ -3,16 +3,159 @@ require 'spec_helper' RSpec.describe Gitlab::Graphql::Subscriptions::ActionCableWithLoadBalancing, feature_category: :shared do - let(:session_class) { ::Gitlab::Database::LoadBalancing::Session } - let(:session) { instance_double(session_class) } - let(:event) { instance_double(::GraphQL::Subscriptions::Event) } + let(:field) { Types::SubscriptionType.fields.each_value.first } + let(:event) { ::GraphQL::Subscriptions::Event.new(name: 'test-event', arguments: {}, field: field) } + let(:object) { build(:project, id: 1) } + let(:action_cable) { instance_double(::ActionCable::Server::Broadcasting) } subject(:subscriptions) { described_class.new(schema: GitlabSchema) } - it 'forces use of DB primary when executing subscription updates' do - expect(session_class).to receive(:current).and_return(session) - expect(session).to receive(:use_primary!) + include_context 'when tracking WAL location reference' - subscriptions.execute_update('sub:123', event, {}) + before do + allow(::ActionCable).to receive(:server).and_return(action_cable) + end + + context 'when triggering subscription' do + shared_examples_for 'injecting WAL locations' do + it 'injects correct WAL location into message' do + expect(action_cable).to receive(:broadcast) do |topic, payload| + expect(topic).to match(/^graphql-event/) + expect(Gitlab::Json.parse(payload)).to match({ + 'wal_locations' => expected_locations, + 'payload' => { '__gid__' => be_instance_of(String) } + }) + end + + subscriptions.execute_all(event, object) + end + end + + context 'when feature is disabled' do + before do + stub_feature_flags(graphql_subs_lb: false) + end + + it 'does not inject WAL locations' do + expect(action_cable).to receive(:broadcast) do |topic, payload| + expect(topic).to match(/^graphql-event/) + expect(Gitlab::Json.parse(payload)).to match({ '__gid__' => be_instance_of(String) }) + end + + subscriptions.execute_all(event, object) + end + end + + context 'when database load balancing is disabled' do + let!(:expected_locations) { {} } + + before do + stub_load_balancing_disabled! + end + + it_behaves_like 'injecting WAL locations' + end + + context 'when database load balancing is enabled' do + before do + stub_load_balancing_enabled! + end + + context 'when write was not performed' do + before do + stub_no_writes_performed! + end + + context 'when replica hosts are available' do + let!(:expected_locations) { expect_tracked_locations_when_replicas_available.with_indifferent_access } + + it_behaves_like 'injecting WAL locations' + end + + context 'when no replica hosts are available' do + let!(:expected_locations) { expect_tracked_locations_when_no_replicas_available.with_indifferent_access } + + it_behaves_like 'injecting WAL locations' + end + end + + context 'when write was performed' do + let!(:expected_locations) { expect_tracked_locations_from_primary_only.with_indifferent_access } + + before do + stub_write_performed! + end + + it_behaves_like 'injecting WAL locations' + end + end + end + + context 'when handling event' do + def handle_event! + subscriptions.execute_update('sub:123', event, object) + end + + before do + allow(action_cable).to receive(:broadcast) + + subscriptions.load_action_cable_message(Gitlab::Json.dump({ + 'wal_locations' => { + 'main' => current_location + }, + 'payload' => {} + }), nil) + end + + context 'when feature is disabled' do + before do + stub_feature_flags(graphql_subs_lb: false) + end + + it 'uses the primary' do + expect(::Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary!) + + handle_event! + end + end + + context 'when WAL locations are not present' do + it 'uses the primary' do + subscriptions.load_action_cable_message(Gitlab::Json.dump({}), nil) + + expect(::Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary!) + + handle_event! + end + end + + it 'strips out WAL location information before broadcasting payload' do + expect(action_cable).to receive(:broadcast) do |topic, payload| + expect(topic).to eq('graphql-subscription:sub:123') + expect(payload).to eq({ more: false }) + end + + handle_event! + end + + context 'when database replicas are in sync' do + it 'does not use the primary' do + stub_replica_available!(true) + + expect(::Gitlab::Database::LoadBalancing::Session.current).not_to receive(:use_primary!) + + handle_event! + end + end + + context 'when database replicas are not in sync' do + it 'uses the primary' do + stub_replica_available!(false) + + expect(::Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary!) + + handle_event! + end + end end end diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index 64411c9fe92..7b5c3c82b7f 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -277,6 +277,13 @@ RSpec.describe ApplicationSetting, feature_category: :shared, type: :model do it { is_expected.to allow_value([true, false]).for(:silent_mode_enabled) } it { is_expected.not_to allow_value(nil).for(:silent_mode_enabled) } + it { is_expected.to allow_value(0).for(:ci_max_includes) } + it { is_expected.to allow_value(200).for(:ci_max_includes) } + it { is_expected.not_to allow_value('abc').for(:ci_max_includes) } + it { is_expected.not_to allow_value(nil).for(:ci_max_includes) } + it { is_expected.not_to allow_value(10.5).for(:ci_max_includes) } + it { is_expected.not_to allow_value(-1).for(:ci_max_includes) } + context 'when deactivate_dormant_users is enabled' do before do stub_application_setting(deactivate_dormant_users: true) diff --git a/spec/requests/api/settings_spec.rb b/spec/requests/api/settings_spec.rb index ff86859a8f4..8decb8cdf90 100644 --- a/spec/requests/api/settings_spec.rb +++ b/spec/requests/api/settings_spec.rb @@ -70,6 +70,7 @@ RSpec.describe API::Settings, 'Settings', :do_not_mock_admin_mode_setting, featu expect(json_response['projects_api_rate_limit_unauthenticated']).to eq(400) expect(json_response['silent_mode_enabled']).to be(false) expect(json_response['valid_runner_registrars']).to match_array(%w(project group)) + expect(json_response['ci_max_includes']).to eq(150) end end @@ -819,6 +820,40 @@ RSpec.describe API::Settings, 'Settings', :do_not_mock_admin_mode_setting, featu end end + context 'with ci_max_includes' do + it 'updates the settings' do + put api("/application/settings", admin), params: { + ci_max_includes: 200 + } + + expect(response).to have_gitlab_http_status(:ok) + expect(json_response).to include( + 'ci_max_includes' => 200 + ) + end + + it 'allows a zero value' do + put api("/application/settings", admin), params: { + ci_max_includes: 0 + } + + expect(response).to have_gitlab_http_status(:ok) + expect(json_response).to include( + 'ci_max_includes' => 0 + ) + end + + it 'does not allow a nil value' do + put api("/application/settings", admin), params: { + ci_max_includes: nil + } + + expect(response).to have_gitlab_http_status(:bad_request) + expect(json_response['message']['ci_max_includes']) + .to include(a_string_matching('is not a number')) + end + end + context 'with housekeeping enabled' do it 'at least one of housekeeping_incremental_repack_period or housekeeping_optimize_repository_period is required' do put api("/application/settings", admin), params: { diff --git a/spec/support/helpers/features/mirroring_helpers.rb b/spec/support/helpers/features/mirroring_helpers.rb new file mode 100644 index 00000000000..0c3006cd1d1 --- /dev/null +++ b/spec/support/helpers/features/mirroring_helpers.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# These helpers allow you to set up mirroring. +# +# Usage: +# describe "..." do +# include Features::MirroringHelpers +# ... +# +# fill_and_wait_for_mirror_url_javascript("url", "ssh://user@localhost/project.git") +# wait_for_mirror_field_javascript("protected", "0") +# +module Features + module MirroringHelpers + # input_identifier - identifier of the input field, passed to `fill_in` (can be an ID or a label). + # url - the URL to fill the input field with. + def fill_and_wait_for_mirror_url_javascript(input_identifier, url) + fill_in input_identifier, with: url + wait_for_mirror_field_javascript('url', url) + end + + # attribute - can be `url` or `protected`. It's used in the `.js-mirror-<field>-hidden` selector. + # expected_value - the expected value of the hidden field. + def wait_for_mirror_field_javascript(attribute, expected_value) + expect(page).to have_css(".js-mirror-#{attribute}-hidden[value=\"#{expected_value}\"]", visible: :hidden) + end + end +end diff --git a/spec/support/shared_contexts/lib/gitlab/database/load_balancing/wal_tracking_shared_context.rb b/spec/support/shared_contexts/lib/gitlab/database/load_balancing/wal_tracking_shared_context.rb new file mode 100644 index 00000000000..cbbd3754108 --- /dev/null +++ b/spec/support/shared_contexts/lib/gitlab/database/load_balancing/wal_tracking_shared_context.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +RSpec.shared_context 'when tracking WAL location reference' do + let(:current_location) { '0/D525E3A8' } + + around do |example| + Gitlab::Database::LoadBalancing::Session.clear_session + example.run + Gitlab::Database::LoadBalancing::Session.clear_session + end + + def expect_tracked_locations_when_replicas_available + {}.tap do |locations| + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + expect(lb.host).to receive(:database_replica_location).and_return(current_location) + + locations[lb.name] = current_location + end + end + end + + def expect_tracked_locations_when_no_replicas_available + {}.tap do |locations| + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + expect(lb).to receive(:host).at_least(:once).and_return(nil) + expect(lb).to receive(:primary_write_location).and_return(current_location) + + locations[lb.name] = current_location + end + end + end + + def expect_tracked_locations_from_primary_only + {}.tap do |locations| + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + expect(lb).to receive(:primary_write_location).and_return(current_location) + + locations[lb.name] = current_location + end + end + end + + def stub_load_balancing_disabled! + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + allow(lb).to receive(:primary_only?).and_return(true) + end + end + + def stub_load_balancing_enabled! + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + allow(lb).to receive(:primary_only?).and_return(false) + end + end + + def stub_no_writes_performed! + allow(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary?).and_return(false) + end + + def stub_write_performed! + allow(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary?).and_return(true) + end + + def stub_replica_available!(available) + ::Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + allow(lb).to receive(:select_up_to_date_host).with(current_location).and_return(available) + end + end +end diff --git a/spec/workers/every_sidekiq_worker_spec.rb b/spec/workers/every_sidekiq_worker_spec.rb index 0a4dd38de13..5778bc47b35 100644 --- a/spec/workers/every_sidekiq_worker_spec.rb +++ b/spec/workers/every_sidekiq_worker_spec.rb @@ -262,7 +262,6 @@ RSpec.describe 'Every Sidekiq worker', feature_category: :shared do 'GeoRepositoryDestroyWorker' => 3, 'GitGarbageCollectWorker' => false, 'Gitlab::GithubImport::AdvanceStageWorker' => 3, - 'Gitlab::GithubImport::ImportReleaseAttachmentsWorker' => 5, 'Gitlab::GithubImport::Attachments::ImportReleaseWorker' => 5, 'Gitlab::GithubImport::Attachments::ImportNoteWorker' => 5, 'Gitlab::GithubImport::Attachments::ImportIssueWorker' => 5, diff --git a/spec/workers/gitlab/github_import/import_release_attachments_worker_spec.rb b/spec/workers/gitlab/github_import/import_release_attachments_worker_spec.rb deleted file mode 100644 index 62a9e3446f8..00000000000 --- a/spec/workers/gitlab/github_import/import_release_attachments_worker_spec.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::GithubImport::ImportReleaseAttachmentsWorker, feature_category: :importers do - subject(:worker) { described_class.new } - - describe '#import' do - let(:import_state) { create(:import_state, :started) } - - let(:project) do - instance_double('Project', full_path: 'foo/bar', id: 1, import_state: import_state) - end - - let(:client) { instance_double('Gitlab::GithubImport::Client') } - let(:importer) { instance_double('Gitlab::GithubImport::Importer::NoteAttachmentsImporter') } - - let(:release_hash) do - { - 'record_db_id' => rand(100), - 'record_type' => 'Release', - 'tag' => 'v1.0', - 'text' => <<-TEXT - Some text... - -  - TEXT - } - end - - it 'imports an issue event' do - expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter) - .to receive(:new) - .with( - an_instance_of(Gitlab::GithubImport::Representation::NoteText), - project, - client - ) - .and_return(importer) - - expect(importer).to receive(:execute) - - expect(Gitlab::GithubImport::ObjectCounter) - .to receive(:increment) - .and_call_original - - worker.import(project, client, release_hash) - end - end -end diff --git a/yarn.lock b/yarn.lock index fbf5b697039..f105441ac67 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1110,15 +1110,15 @@ stylelint-declaration-strict-value "1.8.0" stylelint-scss "4.2.0" -"@gitlab/svgs@3.40.0": - version "3.40.0" - resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.40.0.tgz#f1ebb2fcdbb1181550d53f0db827eca1f5060af0" - integrity sha512-9CVkIbV0VnIFfVBjWcW8+nHzpMhHhC73C9mGPEktEPfpEbaaRws2UywgDEH+C2B8Ba1QdBo/aFr68RDu2VwvfA== - -"@gitlab/ui@62.4.0": - version "62.4.0" - resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-62.4.0.tgz#827e2506a2880fb8d9899b9c18c1d5dfae79a9a7" - integrity sha512-FouJu0o3Za2Hz6WjnZSN+LTMSjpaBCE3SZ5HSBoz0be/WKjS3CX5WW3Qp2NMvNs59xe4V0xg3uWI2g/X0edjTw== +"@gitlab/svgs@3.42.0": + version "3.42.0" + resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.42.0.tgz#3168eb46ed9f87abc02b749cc1468902aee8e964" + integrity sha512-hOZtSWKndBP7WO7UxOe+dlMtWK+4J9srLBEAro8ZNcRufm+esF8N8HBPgFvx4ef6GVAW7Ej7qe1wW9kiQQGDyQ== + +"@gitlab/ui@62.5.0": + version "62.5.0" + resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-62.5.0.tgz#f8038a27a742910418dc48b47020af75a5beaf02" + integrity sha512-JIWzFUV3n0LfqmR5fZ6mqFHqyZ+SlqnhnS74/3weqNos63PQ6CmXe58qW7LxMPj5+B9HWm76Hwd3KEHvv5Q1Aw== dependencies: "@popperjs/core" "^2.11.2" bootstrap-vue "2.23.1" |