diff options
49 files changed, 523 insertions, 310 deletions
diff --git a/app/assets/javascripts/issuable/components/issuable_header_warnings.vue b/app/assets/javascripts/issuable/components/issuable_header_warnings.vue index 7f313086fca..06d1a2ee233 100644 --- a/app/assets/javascripts/issuable/components/issuable_header_warnings.vue +++ b/app/assets/javascripts/issuable/components/issuable_header_warnings.vue @@ -63,8 +63,11 @@ export default { v-gl-tooltip :data-testid="meta.dataTestId" :title="meta.tooltip || null" - :class="{ 'gl-mr-3 gl-mt-2': isMergeRequest }" - class="issuable-warning-icon gl-display-flex gl-justify-content-center gl-align-items-center" + :class="{ + 'gl-mr-3 gl-mt-2 gl-display-flex gl-justify-content-center gl-align-items-center': isMergeRequest, + 'gl-display-inline-block': !isMergeRequest, + }" + class="issuable-warning-icon" > <gl-icon :name="meta.iconName" class="icon" /> </div> diff --git a/app/assets/javascripts/lib/dompurify.js b/app/assets/javascripts/lib/dompurify.js index 47568f0ecff..4959550e273 100644 --- a/app/assets/javascripts/lib/dompurify.js +++ b/app/assets/javascripts/lib/dompurify.js @@ -7,6 +7,7 @@ const defaultConfig = { // Prevent possible XSS attacks with data-* attributes used by @rails/ujs // See https://gitlab.com/gitlab-org/gitlab-ui/-/issues/1421 FORBID_ATTR: ['data-remote', 'data-url', 'data-type', 'data-method'], + FORBID_TAGS: ['style', 'mstyle'], }; // Only icons urls from `gon` are allowed diff --git a/app/assets/javascripts/lib/utils/url_utility.js b/app/assets/javascripts/lib/utils/url_utility.js index 335cd6a16e5..ff60fd2aecb 100644 --- a/app/assets/javascripts/lib/utils/url_utility.js +++ b/app/assets/javascripts/lib/utils/url_utility.js @@ -420,6 +420,19 @@ export function isSafeURL(url) { } /** + * Returns the sanitized url when not safe + * + * @param {String} url + * @returns {String} + */ +export function sanitizeUrl(url) { + if (!isSafeURL(url)) { + return 'about:blank'; + } + return url; +} + +/** * Returns a normalized url * * https://gitlab.com/foo/../baz => https://gitlab.com/baz diff --git a/app/assets/javascripts/project_select_combo_button.js b/app/assets/javascripts/project_select_combo_button.js index 09dbf2cee04..ad80032c551 100644 --- a/app/assets/javascripts/project_select_combo_button.js +++ b/app/assets/javascripts/project_select_combo_button.js @@ -1,5 +1,6 @@ import $ from 'jquery'; import { sprintf, __ } from '~/locale'; +import { sanitizeUrl } from '~/lib/utils/url_utility'; import AccessorUtilities from './lib/utils/accessor'; import { loadCSSFile } from './lib/utils/css_utils'; @@ -80,7 +81,7 @@ export default class ProjectSelectComboButton { setNewItemBtnAttributes(project) { if (project) { - this.newItemBtn.attr('href', project.url); + this.newItemBtn.attr('href', sanitizeUrl(project.url)); this.newItemBtn.text( sprintf(__('New %{type} in %{project}'), { type: this.resourceLabel, diff --git a/app/controllers/projects/service_ping_controller.rb b/app/controllers/projects/service_ping_controller.rb index d8e3990a244..d8f1785d95e 100644 --- a/app/controllers/projects/service_ping_controller.rb +++ b/app/controllers/projects/service_ping_controller.rb @@ -3,7 +3,7 @@ class Projects::ServicePingController < Projects::ApplicationController before_action :authenticate_user! - feature_category :service_ping + feature_category :web_ide def web_ide_clientside_preview return render_404 unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index f6fff68e98b..b3477fa0e3e 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -407,6 +407,8 @@ module ApplicationSettingsHelper :container_registry_import_max_retries, :container_registry_import_start_max_retries, :container_registry_import_max_step_duration, + :container_registry_pre_import_timeout, + :container_registry_import_timeout, :container_registry_import_target_plan, :container_registry_import_created_before, :keep_latest_artifact, diff --git a/app/helpers/merge_requests_helper.rb b/app/helpers/merge_requests_helper.rb index 29b2f2eb12f..f167734034d 100644 --- a/app/helpers/merge_requests_helper.rb +++ b/app/helpers/merge_requests_helper.rb @@ -246,13 +246,13 @@ module MergeRequestsHelper '' end - link_to branch, branch_path, class: 'gl-link gl-font-monospace gl-bg-blue-50 gl-rounded-base gl-font-sm gl-p-2 gl-display-inline-block gl-text-truncate gl-w-30p gl-mb-n3' + link_to branch, branch_path, class: 'gl-link gl-font-monospace gl-bg-blue-50 gl-rounded-base gl-font-sm gl-p-2 gl-display-inline-block gl-text-truncate gl-max-w-26 gl-mb-n3' end def merge_request_header(project, merge_request) link_to_author = link_to_member(project, merge_request.author, size: 24, extra_class: 'gl-font-weight-bold', avatar: false) copy_button = clipboard_button(text: merge_request.source_branch, title: _('Copy branch name'), class: 'btn btn-default btn-sm gl-button btn-default-tertiary btn-icon gl-display-none! gl-md-display-inline-block! js-source-branch-copy') - target_branch = link_to merge_request.target_branch, project_tree_path(merge_request.target_project, merge_request.target_branch), class: 'gl-link gl-font-monospace gl-bg-blue-50 gl-rounded-base gl-font-sm gl-p-2 gl-display-inline-block gl-text-truncate gl-w-20p gl-mb-n3' + target_branch = link_to merge_request.target_branch, project_tree_path(merge_request.target_project, merge_request.target_branch), class: 'gl-link gl-font-monospace gl-bg-blue-50 gl-rounded-base gl-font-sm gl-p-2 gl-display-inline-block gl-text-truncate gl-max-w-26 gl-mb-n3' _('%{author} requested to merge %{source_branch} %{copy_button} into %{target_branch} %{created_at}').html_safe % { author: link_to_author.html_safe, source_branch: merge_request_source_branch(merge_request).html_safe, copy_button: copy_button.html_safe, target_branch: target_branch.html_safe, created_at: time_ago_with_tooltip(merge_request.created_at, html_class: 'gl-display-inline-block').html_safe } end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index bf68101934b..d40f3784fa1 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -377,6 +377,8 @@ class ApplicationSetting < ApplicationRecord :container_registry_import_max_retries, :container_registry_import_start_max_retries, :container_registry_import_max_step_duration, + :container_registry_pre_import_timeout, + :container_registry_import_timeout, allow_nil: false, numericality: { only_integer: true, greater_than_or_equal_to: 0 } diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 36366b375b9..a54dc4f691d 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -224,6 +224,8 @@ module ApplicationSettingImplementation container_registry_import_max_retries: 3, container_registry_import_start_max_retries: 50, container_registry_import_max_step_duration: 5.minutes, + container_registry_pre_import_timeout: 30.minutes, + container_registry_import_timeout: 10.minutes, container_registry_import_target_plan: 'free', container_registry_import_created_before: '2022-01-23 00:00:00', kroki_enabled: false, diff --git a/app/services/merge_requests/post_merge_service.rb b/app/services/merge_requests/post_merge_service.rb index e475b57e4a2..980c757bcbc 100644 --- a/app/services/merge_requests/post_merge_service.rb +++ b/app/services/merge_requests/post_merge_service.rb @@ -45,7 +45,19 @@ module MergeRequests closed_issues = merge_request.visible_closing_issues_for(current_user) closed_issues.each do |issue| - Issues::CloseService.new(project: project, current_user: current_user).execute(issue, commit: merge_request) + # We are intentionally only closing Issues asynchronously (excluding ExternalIssues) + # as the worker only supports finding an Issue. We are also only experiencing + # SQL timeouts when closing an Issue. + if Feature.enabled?(:async_mr_close_issue, project) && issue.is_a?(Issue) + MergeRequests::CloseIssueWorker.perform_async( + project.id, + current_user.id, + issue.id, + merge_request.id + ) + else + Issues::CloseService.new(project: project, current_user: current_user).execute(issue, commit: merge_request) + end end end diff --git a/app/views/admin/application_settings/_prometheus.html.haml b/app/views/admin/application_settings/_prometheus.html.haml index e2051e17bf9..59681c0278e 100644 --- a/app/views/admin/application_settings/_prometheus.html.haml +++ b/app/views/admin/application_settings/_prometheus.html.haml @@ -1,20 +1,17 @@ -= form_for @application_setting, url: metrics_and_profiling_admin_application_settings_path(anchor: 'js-prometheus-settings'), html: { class: 'fieldset-form' } do |f| += gitlab_ui_form_for @application_setting, url: metrics_and_profiling_admin_application_settings_path(anchor: 'js-prometheus-settings'), html: { class: 'fieldset-form' } do |f| = form_errors(@application_setting) %fieldset .form-group - .form-check - = f.check_box :prometheus_metrics_enabled, class: 'form-check-input' - = f.label :prometheus_metrics_enabled, class: 'form-check-label' do - = _("Enable health and performance metrics endpoint") - .form-text.text-muted - = _('Enable collection of application metrics. Restart required.') - = link_to _('How to export these metrics to Prometheus?'), help_page_path('administration/monitoring/prometheus/gitlab_metrics.md'), target: '_blank', rel: 'noopener noreferrer' + - prometheus_help_link_url = help_page_path('administration/monitoring/prometheus/gitlab_metrics') + - prometheus_help_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: prometheus_help_link_url } + = f.gitlab_ui_checkbox_component :prometheus_metrics_enabled, + _('Enable health and performance metrics endpoint'), + help_text: s_('AdminSettings|Enable a Prometheus endpoint that exposes health and performance statistics. The Health Check menu item appears in the Monitoring section of the Admin Area. Restart required. %{link_start}Learn more.%{link_end}').html_safe % { link_start: prometheus_help_link_start, link_end: '</a>'.html_safe } + .form-text.gl-text-gray-500.gl-pl-6 - unless Gitlab::Metrics.metrics_folder_present? - .form-text.text-muted - %strong.cred= _("WARNING:") - = _("Environment variable %{environment_variable} does not exist or is not pointing to a valid directory.").html_safe % { environment_variable: '<code>prometheus_multiproc_dir</code>'.html_safe } - = link_to sprite_icon('question-o'), help_page_path('administration/monitoring/prometheus/gitlab_metrics', anchor: 'metrics-shared-directory') + - icon_link = link_to sprite_icon('question-o'), help_page_path('administration/monitoring/prometheus/gitlab_metrics', anchor: 'metrics-shared-directory'), target: '_blank', rel: 'noopener noreferrer' + = s_('AdminSettings|%{strongStart}WARNING:%{strongEnd} Environment variable %{environment_variable} does not exist or is not pointing to a valid directory. %{icon_link}').html_safe % { strongStart: '<strong class="gl-text-red-500">'.html_safe, strongEnd: '</strong>'.html_safe, environment_variable: '<code>prometheus_multiproc_dir</code>'.html_safe, icon_link: icon_link } .form-group = f.label :metrics_method_call_threshold, _('Method call threshold (ms)'), class: 'label-bold' = f.number_field :metrics_method_call_threshold, class: 'form-control gl-form-input' diff --git a/app/views/admin/background_migrations/index.html.haml b/app/views/admin/background_migrations/index.html.haml index b2b66a94970..e3ef2587e5d 100644 --- a/app/views/admin/background_migrations/index.html.haml +++ b/app/views/admin/background_migrations/index.html.haml @@ -1,7 +1,7 @@ - page_title s_('BackgroundMigrations|Background Migrations') .gl-display-flex.gl-sm-flex-direction-column.gl-sm-align-items-flex-end.gl-pb-5.gl-border-b-1.gl-border-b-solid.gl-border-b-gray-100 - .gl-flex-grow-1.gl-mr-7 + .gl-flex-grow-1 %h3= s_('BackgroundMigrations|Background Migrations') %p.light.gl-mb-0 - learnmore_link = help_page_path('development/database/batched_background_migrations') @@ -9,7 +9,7 @@ = html_escape(s_('BackgroundMigrations|Background migrations are used to perform data migrations whenever a migration exceeds the time limits in our guidelines. %{linkStart}Learn more%{linkEnd}')) % { linkStart: learnmore_link_start, linkEnd: '</a>'.html_safe } - if @databases.size > 1 - .gl-display-flex.gl-align-items-center.gl-flex-grow-0.gl-flex-basis-0.gl-sm-mt-0.gl-mt-5 + .gl-display-flex.gl-align-items-center.gl-flex-grow-0.gl-flex-basis-0.gl-sm-mt-0.gl-mt-5.gl-sm-ml-7.gl-ml-0 #js-database-listbox{ data: { databases: @databases, selected_database: @selected_database } } = gl_tabs_nav do diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index c436335a244..17866ef7296 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -2506,6 +2506,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: merge_requests_close_issue + :worker_name: MergeRequests::CloseIssueWorker + :feature_category: :code_review + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: merge_requests_delete_source_branch :worker_name: MergeRequests::DeleteSourceBranchWorker :feature_category: :source_code_management diff --git a/app/workers/container_registry/migration/guard_worker.rb b/app/workers/container_registry/migration/guard_worker.rb index e5b3bd908c1..cb74360d2d6 100644 --- a/app/workers/container_registry/migration/guard_worker.rb +++ b/app/workers/container_registry/migration/guard_worker.rb @@ -64,7 +64,17 @@ module ContainerRegistry end def long_running_migration?(repository) - migration_start_timestamp(repository).before?(long_running_migration_threshold) + timeout = long_running_migration_threshold + + if Feature.enabled?(:registry_migration_guard_thresholds) + timeout = if repository.migration_state == 'pre_importing' + migration.pre_import_timeout + else + migration.import_timeout + end + end + + migration_start_timestamp(repository).before?(timeout.ago) end def external_state_matches_migration_state?(repository) @@ -83,17 +93,21 @@ module ContainerRegistry end def step_before_timestamp - ::ContainerRegistry::Migration.max_step_duration.seconds.ago + migration.max_step_duration.seconds.ago end def max_capacity # doubling the actual capacity to prevent issues in case the capacity # is not properly applied - ::ContainerRegistry::Migration.capacity * 2 + migration.capacity * 2 + end + + def migration + ::ContainerRegistry::Migration end def long_running_migration_threshold - @threshold ||= 10.minutes.ago + @threshold ||= 10.minutes end def cancel_long_running_migration(repository) diff --git a/app/workers/merge_requests/close_issue_worker.rb b/app/workers/merge_requests/close_issue_worker.rb new file mode 100644 index 00000000000..86d63e571ac --- /dev/null +++ b/app/workers/merge_requests/close_issue_worker.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module MergeRequests + class CloseIssueWorker + include ApplicationWorker + + data_consistency :always + feature_category :code_review + urgency :low + idempotent! + + # Issues:CloseService execute webhooks which are treated as external dependencies + worker_has_external_dependencies! + + # This worker only accepts ID of an Issue. We are intentionally using this + # worker to close Issues asynchronously as we only experience SQL timeouts + # when closing an Issue. + def perform(project_id, user_id, issue_id, merge_request_id) + project = Project.find_by_id(project_id) + + unless project + logger.info(structured_payload(message: 'Project not found.', project_id: project_id)) + return + end + + user = User.find_by_id(user_id) + + unless user + logger.info(structured_payload(message: 'User not found.', user_id: user_id)) + return + end + + issue = Issue.find_by_id(issue_id) + + unless issue + logger.info(structured_payload(message: 'Issue not found.', issue_id: issue_id)) + return + end + + merge_request = MergeRequest.find_by_id(merge_request_id) + + unless merge_request + logger.info(structured_payload(message: 'Merge request not found.', merge_request_id: merge_request_id)) + return + end + + Issues::CloseService + .new(project: project, current_user: user) + .execute(issue, commit: merge_request) + end + end +end diff --git a/config/feature_flags/development/async_mr_close_issue.yml b/config/feature_flags/development/async_mr_close_issue.yml new file mode 100644 index 00000000000..84db0a57fd6 --- /dev/null +++ b/config/feature_flags/development/async_mr_close_issue.yml @@ -0,0 +1,8 @@ +--- +name: async_mr_close_issue +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86328 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/361320 +milestone: '15.0' +type: development +group: group::code review +default_enabled: false diff --git a/config/feature_flags/development/registry_migration_guard_thresholds.yml b/config/feature_flags/development/registry_migration_guard_thresholds.yml new file mode 100644 index 00000000000..664897c9e87 --- /dev/null +++ b/config/feature_flags/development/registry_migration_guard_thresholds.yml @@ -0,0 +1,8 @@ +--- +name: registry_migration_guard_thresholds +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/360790 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/350543 +milestone: '15.0' +type: development +group: group::package +default_enabled: false diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index f3df7608052..104580761d0 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -263,6 +263,8 @@ - 1 - - merge_request_reset_approvals - 1 +- - merge_requests_close_issue + - 1 - - merge_requests_delete_source_branch - 1 - - merge_requests_handle_assignees_change diff --git a/data/removals/15_0/15-0-request-profiling.yml b/data/removals/15_0/15-0-request-profiling.yml index b5f7c39b8bd..0ca6919130c 100644 --- a/data/removals/15_0/15-0-request-profiling.yml +++ b/data/removals/15_0/15-0-request-profiling.yml @@ -1,6 +1,6 @@ - name: "Request profiling" announcement_milestone: "14.8" - announcement_date: "2021-02-22" + announcement_date: "2022-02-22" removal_milestone: "15.0" removal_date: "2022-05-22" breaking_change: true diff --git a/db/migrate/20220511191502_add_registry_migration_guard_thresholds_to_application_settings.rb b/db/migrate/20220511191502_add_registry_migration_guard_thresholds_to_application_settings.rb new file mode 100644 index 00000000000..87a29bd5889 --- /dev/null +++ b/db/migrate/20220511191502_add_registry_migration_guard_thresholds_to_application_settings.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddRegistryMigrationGuardThresholdsToApplicationSettings < Gitlab::Database::Migration[2.0] + def change + add_column :application_settings, :container_registry_pre_import_timeout, + :integer, + default: 30.minutes, + null: false + + add_column :application_settings, :container_registry_import_timeout, + :integer, + default: 10.minutes, + null: false + end +end diff --git a/db/schema_migrations/20220511191502 b/db/schema_migrations/20220511191502 new file mode 100644 index 00000000000..caebd721aa6 --- /dev/null +++ b/db/schema_migrations/20220511191502 @@ -0,0 +1 @@ +432214f4683800e1f5b5e42d05d9a6de07c317fec0dffd6b1eb312ccfd437e0c
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 19de1d7ba2f..61a95095e34 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -11292,6 +11292,8 @@ CREATE TABLE application_settings ( delayed_group_deletion boolean DEFAULT true NOT NULL, arkose_labs_namespace text DEFAULT 'client'::text NOT NULL, max_export_size integer DEFAULT 0, + container_registry_pre_import_timeout integer DEFAULT 1800 NOT NULL, + container_registry_import_timeout integer DEFAULT 600 NOT NULL, CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)), CONSTRAINT app_settings_dep_proxy_ttl_policies_worker_capacity_positive CHECK ((dependency_proxy_ttl_group_policy_worker_capacity >= 0)), CONSTRAINT app_settings_ext_pipeline_validation_service_url_text_limit CHECK ((char_length(external_pipeline_validation_service_url) <= 255)), diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index cc7daccfa5b..0f4fa1a97a8 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -290,9 +290,6 @@ useful when debugging. The default value for `SECURE_LOG_LEVEL` should be set to `info`. When executing command lines, scanners should use the `debug` level to log the command line and its output. -For instance, the [bundler-audit](https://gitlab.com/gitlab-org/security-products/analyzers/bundler-audit) scanner -uses the `debug` level to log the command line `bundle audit check --quiet`, -and what `bundle audit` writes to the standard output. If the command line fails, then it should be logged with the `error` log level; this makes it possible to debug the problem without having to change the log level to `debug` and rerun the scanning job. diff --git a/doc/topics/autodevops/customize.md b/doc/topics/autodevops/customize.md index f84abca1ff2..4f797f5c238 100644 --- a/doc/topics/autodevops/customize.md +++ b/doc/topics/autodevops/customize.md @@ -479,7 +479,6 @@ The following table lists variables used to disable jobs. | `build_artifact` | `BUILD_DISABLED` | | If the variable is present, the job isn't created. | | `bandit-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | | `brakeman-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `bundler-audit-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | | `canary` | `CANARY_ENABLED` | | This manual job is created if the variable is present. | | `cluster_image_scanning` | `CLUSTER_IMAGE_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | | `code_intelligence` | `CODE_INTELLIGENCE_DISABLED` | From GitLab 13.6 | If the variable is present, the job isn't created. | @@ -503,7 +502,6 @@ The following table lists variables used to disable jobs. | `browser_performance` | `BROWSER_PERFORMANCE_DISABLED` | From GitLab 14.0 | Browser performance. If the variable is present, the job isn't created. Replaces `performance`. | | `phpcs-security-audit-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | | `pmd-apex-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `retire-js-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | | `review` | `REVIEW_DISABLED` | | If the variable is present, the job isn't created. | | `review:stop` | `REVIEW_DISABLED` | | Manual job. If the variable is present, the job isn't created. | | `sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | diff --git a/doc/user/application_security/dependency_scanning/analyzers.md b/doc/user/application_security/dependency_scanning/analyzers.md index 665d29c4017..acbc94cba47 100644 --- a/doc/user/application_security/dependency_scanning/analyzers.md +++ b/doc/user/application_security/dependency_scanning/analyzers.md @@ -20,11 +20,9 @@ This is achieved by implementing the [common API](https://gitlab.com/gitlab-org/ Dependency Scanning supports the following official analyzers: -- [`bundler-audit`](https://gitlab.com/gitlab-org/security-products/analyzers/bundler-audit) - [`gemnasium`](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium) - [`gemnasium-maven`](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven) - [`gemnasium-python`](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python) -- [`retire.js`](https://gitlab.com/gitlab-org/security-products/analyzers/retire.js) The analyzers are published as Docker images, which Dependency Scanning uses to launch dedicated containers for each analysis. @@ -34,11 +32,13 @@ The Dependency Scanning analyzers' current major version number is 2. Dependency Scanning is pre-configured with a set of **default images** that are maintained by GitLab, but users can also integrate their own **custom images**. -WARNING: -The `bundler-audit` analyzer is deprecated and will be removed in GitLab 15.0 since it duplicates the functionality of the `gemnasium` analyzer. For more information, read the [deprecation announcement](../../../update/deprecations.md#bundler-audit-dependency-scanning-tool). +<!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> -WARNING: -The `retire.js` analyzer is deprecated and will be removed in GitLab 15.0 since it duplicates the functionality of the `gemnasium` analyzer. For more information, read the [deprecation announcement](../../../update/deprecations.md#retire-js-dependency-scanning-tool). +The [`bundler-audit`](https://gitlab.com/gitlab-org/gitlab/-/issues/289832) and [`retire.js`](https://gitlab.com/gitlab-org/gitlab/-/issues/350510) analyzers were deprecated +in GitLab 14.8 and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86704) in 15.0. +Use Gemnasium instead. + +<!--- end_remove --> ## Official default analyzers @@ -67,7 +67,7 @@ the official analyzers. ### Disable specific analyzers You can select the official analyzers you don't want to run. Here's how to disable -`bundler-audit` and `gemnasium` analyzers. +the `gemnasium` analyzer. In `.gitlab-ci.yml` define: ```yaml @@ -75,7 +75,7 @@ include: template: Security/Dependency-Scanning.gitlab-ci.yml variables: - DS_EXCLUDED_ANALYZERS: "bundler-audit, gemnasium" + DS_EXCLUDED_ANALYZERS: "gemnasium" ``` ### Disabling default analyzers @@ -88,7 +88,7 @@ include: template: Security/Dependency-Scanning.gitlab-ci.yml variables: - DS_EXCLUDED_ANALYZERS: "gemnasium, gemnasium-maven, gemnasium-python, bundler-audit, retire.js" + DS_EXCLUDED_ANALYZERS: "gemnasium, gemnasium-maven, gemnasium-python" ``` This is used when one totally relies on [custom analyzers](#custom-analyzers). @@ -117,25 +117,25 @@ The [Security Scanner Integration](../../../development/integrations/secure.md) ## Analyzers data -The following table lists the data available for each official analyzer. - -| Property \ Tool | Gemnasium | bundler-audit | Retire.js | -|---------------------------------------|:------------------:|:------------------:|:------------------:| -| Severity | 𐄂 | ✓ | ✓ | -| Title | ✓ | ✓ | ✓ | -| File | ✓ | ⚠ | ✓ | -| Start line | 𐄂 | 𐄂 | 𐄂 | -| End line | 𐄂 | 𐄂 | 𐄂 | -| External ID (for example, CVE) | ✓ | ✓ | ⚠ | -| URLs | ✓ | ✓ | ✓ | -| Internal doc/explanation | ✓ | 𐄂 | 𐄂 | -| Solution | ✓ | ✓ | 𐄂 | -| Confidence | 𐄂 | 𐄂 | 𐄂 | -| Affected item (for example, class or package) | ✓ | ✓ | ✓ | -| Source code extract | 𐄂 | 𐄂 | 𐄂 | -| Internal ID | ✓ | 𐄂 | 𐄂 | -| Date | ✓ | 𐄂 | 𐄂 | -| Credits | ✓ | 𐄂 | 𐄂 | +The following table lists the data available for the Gemnasium analyzer. + +| Property \ Tool | Gemnasium | +|---------------------------------------|:------------------:| +| Severity | 𐄂 | +| Title | ✓ | +| File | ✓ | +| Start line | 𐄂 | +| End line | 𐄂 | +| External ID (for example, CVE) | ✓ | +| URLs | ✓ | +| Internal doc/explanation | ✓ | +| Solution | ✓ | +| Confidence | 𐄂 | +| Affected item (for example, class or package) | ✓ | +| Source code extract | 𐄂 | +| Internal ID | ✓ | +| Date | ✓ | +| Credits | ✓ | - ✓ => we have that data - ⚠ => we have that data, but it's partially reliable, or we need to extract that data from unstructured content diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 0cec6750202..909f1353221 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -153,9 +153,9 @@ table.supported-languages ul { </thead> <tbody> <tr> - <td rowspan="2">Ruby</td> - <td rowspan="2">N/A</td> - <td rowspan="2"><a href="https://bundler.io/">Bundler</a></td> + <td>Ruby</td> + <td>N/A</td> + <td><a href="https://bundler.io/">Bundler</a></td> <td> <ul> <li><code>Gemfile.lock</code></li> @@ -166,11 +166,6 @@ table.supported-languages ul { <td>Y</td> </tr> <tr> - <td><code>Gemfile.lock</code></td> - <td><a href="https://github.com/rubysec/bundler-audit">bundler-audit</a></td> - <td>N</td> - </tr> - <tr> <td>PHP</td> <td>N/A</td> <td><a href="https://getcomposer.org/">Composer</a></td> @@ -217,9 +212,9 @@ table.supported-languages ul { <td>N</td> </tr> <tr> - <td rowspan="3">JavaScript</td> - <td rowspan="2">N/A</td> - <td rowspan="2"><a href="https://www.npmjs.com/">npm</a></td> + <td rowspan="2">JavaScript</td> + <td>N/A</td> + <td><a href="https://www.npmjs.com/">npm</a></td> <td> <ul> <li><code>package-lock.json</code></li> @@ -230,11 +225,6 @@ table.supported-languages ul { <td>Y</td> </tr> <tr> - <td><code>package.json</code></td> - <td><a href="https://retirejs.github.io/retire.js/">Retire.js</a></td> - <td>N</td> - </tr> - <tr> <td>N/A</td> <td><a href="https://classic.yarnpkg.com/en/">yarn</a></td> <td><code>yarn.lock</code></td> @@ -355,25 +345,18 @@ To support the following package managers, the GitLab analyzers proceed in two s | Package Manager | Pre-installed Versions | Tested Versions | | ------ | ------ | ------ | -| Bundler | [2.1.4](https://gitlab.com/gitlab-org/security-products/analyzers/bundler-audit/-/blob/v2.11.3/Dockerfile#L15)<sup><b><a href="#exported-dependency-information-notes-1">1</a></b></sup> | [1.17.3](https://gitlab.com/gitlab-org/security-products/tests/ruby-bundler/-/blob/master/Gemfile.lock#L118), [2.1.4](https://gitlab.com/gitlab-org/security-products/tests/ruby-bundler/-/blob/bundler2-FREEZE/Gemfile.lock#L118) | | sbt | [1.6.1](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.24.6/config/.tool-versions#L4) | [1.0.4](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L443-447), [1.1.6](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L449-453), [1.2.8](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L455-459), [1.3.12](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L461-465), [1.4.6](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L467-471), [1.5.8](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L473-477), [1.6.1](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L479-483) | | Maven | [3.6.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L95-97) | [3.6.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L95-97) | -| Gradle | [6.7.1](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.23.0/config/.tool-versions#L5)<sup><b><a href="#exported-dependency-information-notes-2">2</a></b></sup>, [7.3.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.26.0/config/.tool-versions#L5)<sup><b><a href="#exported-dependency-information-notes-2">2</a></b></sup> | [5.6.4](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L319-323), [6.7](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L286-288)<sup><b><a href="#exported-dependency-information-notes-3">3</a></b></sup>, [6.9](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L331-335), [7.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L300-302)<sup><b><a href="#exported-dependency-information-notes-3">3</a></b></sup> | +| Gradle | [6.7.1](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.23.0/config/.tool-versions#L5)<sup><b><a href="#exported-dependency-information-notes-1">1</a></b></sup>, [7.3.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.26.0/config/.tool-versions#L5)<sup><b><a href="#exported-dependency-information-notes-1">1</a></b></sup> | [5.6.4](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L319-323), [6.7](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L286-288)<sup><b><a href="#exported-dependency-information-notes-2">2</a></b></sup>, [6.9](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L331-335), [7.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L300-302)<sup><b><a href="#exported-dependency-information-notes-2">2</a></b></sup> | | setuptools | [50.3.2](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/v2.29.9/Dockerfile#L27) | [57.5.0](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.22.0/spec/image_spec.rb#L224-247) | | pip | [20.2.4](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/v2.29.9/Dockerfile#L26) | [20.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.22.0/spec/image_spec.rb#L77-91) | -| Pipenv | [2018.11.26](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.18.4/requirements.txt#L13) | [2018.11.26](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.22.0/spec/image_spec.rb#L168-191)<sup><b><a href="#exported-dependency-information-notes-4">4</a></b></sup>, [2018.11.26](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.22.0/spec/image_spec.rb#L143-166) | +| Pipenv | [2018.11.26](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.18.4/requirements.txt#L13) | [2018.11.26](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.22.0/spec/image_spec.rb#L168-191)<sup><b><a href="#exported-dependency-information-notes-3">3</a></b></sup>, [2018.11.26](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.22.0/spec/image_spec.rb#L143-166) | <!-- markdownlint-disable MD044 --> <ol> <li> <a id="exported-dependency-information-notes-1"></a> <p> - The pre-installed and tested version of <code>Bundler</code> is only used for the <a href="https://gitlab.com/gitlab-org/security-products/analyzers/bundler-audit">bundler-audit</a> analyzer, and is not used for <a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">gemnasium</a>. - </p> - </li> - <li> - <a id="exported-dependency-information-notes-2"></a> - <p> Different versions of Java require different versions of Gradle. The versions of Gradle listed in the above table are pre-installed in the analyzer image. The version of Gradle used by the analyzer depends on whether your project uses a <code>gradlew</code> (Gradle wrapper) file or not: @@ -400,13 +383,13 @@ To support the following package managers, the GitLab analyzers proceed in two s </ul> </li> <li> - <a id="exported-dependency-information-notes-3"></a> + <a id="exported-dependency-information-notes-2"></a> <p> These tests confirm that if a <code>gradlew</code> file does not exist, the version of <code>Gradle</code> pre-installed in the analyzer image is used. </p> </li> <li> - <a id="exported-dependency-information-notes-4"></a> + <a id="exported-dependency-information-notes-3"></a> <p> This test confirms that if a <code>Pipfile.lock</code> file is found, it will be used by <a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a> to scan the exact package versions listed in this file. </p> @@ -428,22 +411,6 @@ NOTE: If you've run into problems while scanning multiple files, please contribute a comment to [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/337056). -#### Ruby - -The following analyzers are executed, each of which have different behavior when processing multiple files: - -- [Gemnasium](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium) - - Supports multiple lockfiles. - -- [bundler-audit](https://github.com/rubysec/bundler-audit) - - Does not support multiple lockfiles. When multiple lockfiles exist, `bundler-audit` - analyzes the first lockfile discovered while traversing the directory tree in alphabetical order. - -WARNING: -The `bundler-audit` analyzer is deprecated and will be removed in GitLab 15.0 since it duplicates the functionality of the `gemnasium` analyzer. For more information, read the [deprecation announcement](../../../update/deprecations.md#bundler-audit-dependency-scanning-tool). - #### Python We only execute one installation in the directory where a requirements file has been detected, such as `requirements.txt` or any @@ -474,14 +441,10 @@ The following analyzers are executed, each of which have different behavior when Does not support multiple lockfiles. When multiple lockfiles exist, `Retire.js` analyzes the first lockfile discovered while traversing the directory tree in alphabetical order. -From GitLab 14.8 the `Gemnasium` analyzer scans supported JavaScript projects for vendored libraries +From GitLab 14.8 the `gemnasium` analyzer scans supported JavaScript projects for vendored libraries (that is, those checked into the project but not managed by the package manager). -WARNING: -The `retire.js` analyzer is deprecated and will be removed in GitLab 15.0 since it duplicates the functionality of the `gemnasium` analyzer. For more information, read the [deprecation announcement](../../../update/deprecations.md#retire-js-dependency-scanning-tool). -We execute both analyzers because they use different sources of vulnerability data. The result is more comprehensive analysis than if only one was executed. - -#### PHP, Go, C, C++, .NET, C# +#### PHP, Go, C, C++, .NET, C#, Ruby, JavaScript The analyzer for these languages supports multiple lockfiles. @@ -609,9 +572,6 @@ The following variables are used for configuring specific analyzers (used for a | CI/CD variable | Analyzer | Default | Description | |--------------------------------------| ------------------ | ---------------------------- |------------ | -| `BUNDLER_AUDIT_UPDATE_DISABLED` | `bundler-audit` | `"false"` | Disable automatic updates for the `bundler-audit` analyzer. Use if you're running dependency scanning in an offline, air-gapped environment.| -| `BUNDLER_AUDIT_ADVISORY_DB_URL` | `bundler-audit` | `https://github.com/rubysec/ruby-advisory-db` | URL of the advisory database used by bundler-audit. | -| `BUNDLER_AUDIT_ADVISORY_DB_REF_NAME` | `bundler-audit` | `master` | Git ref for the advisory database specified by `BUNDLER_AUDIT_ADVISORY_DB_URL`. | | `GEMNASIUM_DB_LOCAL_PATH` | `gemnasium` | `/gemnasium-db` | Path to local Gemnasium database. | | `GEMNASIUM_DB_UPDATE_DISABLED` | `gemnasium` | `"false"` | Disable automatic updates for the `gemnasium-db` advisory database (For usage see: [examples](#hosting-a-copy-of-the-gemnasium_db-advisory-database))| | `GEMNASIUM_DB_REMOTE_URL` | `gemnasium` | `https://gitlab.com/gitlab-org/security-products/gemnasium-db.git` | Repository URL for fetching the Gemnasium database. | @@ -627,9 +587,6 @@ The following variables are used for configuring specific analyzers (used for a | `PIP_REQUIREMENTS_FILE` | `gemnasium-python` | | Pip requirements file to be scanned. | | `DS_PIP_VERSION` | `gemnasium-python` | | Force the install of a specific pip version (example: `"19.3"`), otherwise the pip installed in the Docker image is used. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12811) in GitLab 12.7) | | `DS_PIP_DEPENDENCY_PATH` | `gemnasium-python` | | Path to load Python pip dependencies from. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12412) in GitLab 12.2) | -| `RETIREJS_JS_ADVISORY_DB` | `retire.js` | `https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/jsrepository.json` | Path or URL to `retire.js` JS vulnerability data file. Note that if the URL hosting the data file uses a custom SSL certificate, for example in an offline installation, you can pass the certificate in the `ADDITIONAL_CA_CERT_BUNDLE` variable. | -| `RETIREJS_NODE_ADVISORY_DB` | `retire.js` | `https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/npmrepository.json` | Path or URL to `retire.js` node vulnerability data file. Note that if the URL hosting the data file uses a custom SSL certificate, for example in an offline installation, you can pass the certificate in the `ADDITIONAL_CA_CERT_BUNDLE` variable. | -| `RETIREJS_ADVISORY_DB_INSECURE` | `retire.js` | `false` | Enable fetching remote JS and Node vulnerability data files (defined by the `RETIREJS_JS_ADVISORY_DB` and `RETIREJS_NODE_ADVISORY_DB` variables) from hosts using an insecure or self-signed SSL (TLS) certificate. | #### Other variables @@ -687,31 +644,11 @@ Read more on [how to use private Maven repositories](../index.md#using-private-m GitLab also offers [FIPS-enabled Red Hat UBI](https://www.redhat.com/en/blog/introducing-red-hat-universal-base-image) versions of the Gemnasium images. You can therefore replace standard images with FIPS-enabled images. -To use FIPS-enabled images, set the `DS_IMAGE_SUFFIX` to `-fips`, -and set `DS_EXCLUDED_ANALYZERS` to `bundler-audit, retire.js` -to exclude the analyzers that don't support FIPS. +To use FIPS-enabled images, set the `DS_IMAGE_SUFFIX` to `-fips`. ```yaml variables: DS_IMAGE_SUFFIX: "-fips" - DS_EXCLUDED_ANALYZERS: "bundler-audit, retire.js" -``` - -If you want to execute `bundler-audit` or `retire.js` in your project pipeline, you can override the -Gemnasium scanning jobs, and set `DS_IMAGE_SUFFIX` to `-fips` only for those jobs. - -```yaml -gemnasium-dependency_scanning: - variables: - DS_IMAGE_SUFFIX: "-fips" - -gemnasium-maven-dependency_scanning: - variables: - DS_IMAGE_SUFFIX: "-fips" - -gemnasium-python-dependency_scanning: - variables: - DS_IMAGE_SUFFIX: "-fips" ``` ## Interacting with the vulnerabilities @@ -964,9 +901,6 @@ Here are the requirements for using dependency scanning in an offline environmen This advisory database is constantly being updated, so you must periodically sync your local copy with GitLab. -- _Only if scanning Ruby projects_: Host an offline Git copy of the [advisory database](https://github.com/rubysec/ruby-advisory-db). -- _Only if scanning npm/yarn projects_: Host an offline copy of the [`retire.js`](https://github.com/RetireJS/retire.js/) [node](https://github.com/RetireJS/retire.js/blob/master/repository/npmrepository.json) and [`js`](https://github.com/RetireJS/retire.js/blob/master/repository/jsrepository.json) advisory databases. - Note that GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), meaning the runner tries to pull Docker images from the GitLab container registry even if a local copy is available. The GitLab Runner [`pull_policy` can be set to `if-not-present`](https://docs.gitlab.com/runner/executors/docker.html#using-the-if-not-present-pull-policy) @@ -984,8 +918,6 @@ your [local Docker container registry](../../packages/container_registry/index.m registry.gitlab.com/security-products/gemnasium:2 registry.gitlab.com/security-products/gemnasium-maven:2 registry.gitlab.com/security-products/gemnasium-python:2 -registry.gitlab.com/security-products/retire.js:2 -registry.gitlab.com/security-products/bundler-audit:2 ``` The process for importing Docker images into a local offline Docker registry depends on @@ -1007,8 +939,6 @@ Support for custom certificate authorities was introduced in the following versi | `gemnasium` | [v2.8.0](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/releases/v2.8.0) | | `gemnasium-maven` | [v2.9.0](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/releases/v2.9.0) | | `gemnasium-python` | [v2.7.0](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/releases/v2.7.0) | -| `retire.js` | [v2.4.0](https://gitlab.com/gitlab-org/security-products/analyzers/retire.js/-/releases/v2.4.0) | -| `bundler-audit` | [v2.4.0](https://gitlab.com/gitlab-org/security-products/analyzers/bundler-audit/-/releases/v2.4.0) | ### Set dependency scanning CI/CD job variables to use local dependency scanning analyzers @@ -1141,22 +1071,6 @@ intended to obtain a private package from a private index. This only affects use requires that the package does not already exist in the public index (and thus the attacker can put the package there with an arbitrary version number). -## Limitations - -### Referencing local dependencies using a path in JavaScript projects - -The [Retire.js](https://gitlab.com/gitlab-org/security-products/analyzers/retire.js) analyzer -doesn't support dependency references made with [local paths](https://docs.npmjs.com/cli/v6/configuring-npm/package-json/#local-paths) -in the `package.json` of JavaScript projects. The dependency scan outputs the following error for -such references: - -```plaintext -ERROR: Could not find dependencies: <dependency-name>. You may need to run npm install -``` - -As a workaround, add the [`retire.js`](analyzers.md) analyzer to -[`DS_EXCLUDED_ANALYZERS`](#configuring-dependency-scanning). - ## Troubleshooting ### Working around missing support for certain languages or package managers @@ -1217,11 +1131,6 @@ syntax. This directive is limited to 10000 checks and always returns `true` afte number. Because of this, and depending on the number of files in your repository, a dependency scanning job might be triggered even if the scanner doesn't support your project. -### Issues building projects with npm or yarn packages relying on Python 2 - -[Python 2 was removed](https://www.python.org/doc/sunset-python-2/) from the `retire.js` analyzer in GitLab 13.7 (analyzer version 2.10.1). Projects using packages -with a dependency on this version of Python should use `retire.js` version 2.10.0 or lower (for example, `registry.gitlab.com/gitlab-org/security-products/analyzers/retire.js:2.10.0`). - ### Error: `dependency_scanning is used for configuration only, and its script should not be executed` For information on this, see the [GitLab Secure troubleshooting section](../index.md#error-job-is-used-for-configuration-only-and-its-script-should-not-be-executed). diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index f5f438dd1ad..7642db947c7 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -49,7 +49,7 @@ The following vulnerability scanners and their databases are regularly updated: | Secure scanning tool | Vulnerabilities database updates | |:----------------------------------------------------------------|:---------------------------------| | [Container Scanning](container_scanning/index.md) | A job runs on a daily basis to build new images with the latest vulnerability database updates from the upstream scanner. For more details, see [Vulnerabilities database update](container_scanning/index.md#vulnerabilities-database-update). | -| [Dependency Scanning](dependency_scanning/index.md) | Relies on `bundler-audit` (for Ruby gems), `retire.js` (for npm packages), and `gemnasium` (the GitLab tool for all libraries). Both `bundler-audit` and `retire.js` fetch their vulnerabilities data from GitHub repositories, so vulnerabilities added to `ruby-advisory-db` and `retire.js` are immediately available. The tools themselves are updated once per month if there's a new version. The [GitLab Advisory Database](https://gitlab.com/gitlab-org/security-products/gemnasium-db) is updated on a daily basis using [data from NVD, the `ruby-advisory-db` and the GitHub Advisory Database as data sources](https://gitlab.com/gitlab-org/security-products/gemnasium-db/-/blob/master/SOURCES.md). See our [current measurement of time from CVE being issued to our product being updated](https://about.gitlab.com/handbook/engineering/development/performance-indicators/#cve-issue-to-update). | +| [Dependency Scanning](dependency_scanning/index.md) | Relies on the [GitLab Advisory Database](https://gitlab.com/gitlab-org/security-products/gemnasium-db). It is updated on a daily basis using [data from NVD, the `ruby-advisory-db` and the GitHub Advisory Database as data sources](https://gitlab.com/gitlab-org/security-products/gemnasium-db/-/blob/master/SOURCES.md). See our [current measurement of time from CVE being issued to our product being updated](https://about.gitlab.com/handbook/engineering/development/performance-indicators/#cve-issue-to-update). | | [Dynamic Application Security Testing (DAST)](dast/index.md) | The scanning engine is updated on a periodic basis. See the [version of the underlying tool `zaproxy`](https://gitlab.com/gitlab-org/security-products/dast/blob/main/Dockerfile#L1). The scanning rules are downloaded at scan runtime. | | [Static Application Security Testing (SAST)](sast/index.md) | Relies exclusively on [the tools GitLab wraps](sast/index.md#supported-languages-and-frameworks). The underlying analyzers are updated at least once per month if a relevant update is available. The vulnerabilities database is updated by the upstream tools. | diff --git a/doc/user/application_security/vulnerabilities/severities.md b/doc/user/application_security/vulnerabilities/severities.md index 89464064ea3..967a6d9fa89 100644 --- a/doc/user/application_security/vulnerabilities/severities.md +++ b/doc/user/application_security/vulnerabilities/severities.md @@ -56,8 +56,6 @@ the following tables: | GitLab analyzer | Outputs severity levels? | Native severity level type | Native severity level example | |------------------------------------------------------------------------------------------|------------------------------|----------------------------|-------------------------------------| -| [`bundler-audit`](https://gitlab.com/gitlab-org/security-products/analyzers/bundler-audit) | **{check-circle}** Yes | String | `low`, `medium`, `high`, `critical` | -| [`retire.js`](https://gitlab.com/gitlab-org/security-products/analyzers/retire.js) | **{check-circle}** Yes | String | `low`, `medium`, `high`, `critical` | | [`gemnasium`](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium) | **{check-circle}** Yes | CVSS v2.0 Rating and CVSS v3.1 Qualitative Severity Rating | `(AV:N/AC:L/Au:S/C:P/I:P/A:N)`, `CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:C/C:H/I:H/A:H` | ## Container Scanning diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index 6c1dc7d072b..4533af459bf 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -332,7 +332,7 @@ after the limits change in January, 2021: | **Authenticated** API traffic (for a given **user**) | **2,000** requests per minute | **2,000** requests per minute | | **Authenticated** non-API HTTP traffic (for a given **user**) | **1,000** requests per minute | **1,000** requests per minute | | **All** traffic (from a given **IP address**) | **2,000** requests per minute | **2,000** requests per minute | -| **Issue creation** | **300** requests per minute | **300** requests per minute | +| **Issue creation** | **300** requests per minute | **200** requests per minute | | **Note creation** (on issues and merge requests) | **60** requests per minute | **60** requests per minute | | **Advanced, project, and group search** API (for a given **IP address**) | **10** requests per minute | **10** requests per minute | | **GitLab Pages** requests (for a given **IP address**) | | **1000** requests per **50 seconds** | diff --git a/lib/container_registry/migration.rb b/lib/container_registry/migration.rb index 33d08e1dec8..c1628e99768 100644 --- a/lib/container_registry/migration.rb +++ b/lib/container_registry/migration.rb @@ -20,6 +20,8 @@ module ContainerRegistry delegate :container_registry_import_max_step_duration, to: ::Gitlab::CurrentSettings delegate :container_registry_import_target_plan, to: ::Gitlab::CurrentSettings delegate :container_registry_import_created_before, to: ::Gitlab::CurrentSettings + delegate :container_registry_pre_import_timeout, to: ::Gitlab::CurrentSettings + delegate :container_registry_import_timeout, to: ::Gitlab::CurrentSettings alias_method :max_tags_count, :container_registry_import_max_tags_count alias_method :max_retries, :container_registry_import_max_retries @@ -27,6 +29,8 @@ module ContainerRegistry alias_method :max_step_duration, :container_registry_import_max_step_duration alias_method :target_plan_name, :container_registry_import_target_plan alias_method :created_before, :container_registry_import_created_before + alias_method :pre_import_timeout, :container_registry_pre_import_timeout + alias_method :import_timeout, :container_registry_import_timeout end def self.enabled? diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index 5b3baebd6fb..f56e017796a 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -125,31 +125,3 @@ gemnasium-python-dependency_scanning: - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $PIP_REQUIREMENTS_FILE - -bundler-audit-dependency_scanning: - extends: .ds-analyzer - variables: - DS_ANALYZER_NAME: "bundler-audit" - rules: - - if: $DEPENDENCY_SCANNING_DISABLED - when: never - - if: $DS_EXCLUDED_ANALYZERS =~ /bundler-audit/ - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ - exists: - - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' - -retire-js-dependency_scanning: - extends: .ds-analyzer - variables: - DS_ANALYZER_NAME: "retire.js" - rules: - - if: $DEPENDENCY_SCANNING_DISABLED - when: never - - if: $DS_EXCLUDED_ANALYZERS =~ /retire.js/ - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ - exists: - - '{package.json,*/package.json,*/*/package.json}' diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index a6fd070ec34..abd9bb4f0dc 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -18,8 +18,7 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SECURE_BINARIES_ANALYZERS: >- - bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kics, kubesec, semgrep, - bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, + bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kics, kubesec, semgrep, gemnasium, gemnasium-maven, gemnasium-python, license-finder, dast, dast-runner-validation, api-fuzzing @@ -174,20 +173,6 @@ kubesec: # Dependency Scanning jobs # -bundler-audit: - extends: .download_images - only: - variables: - - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && - $SECURE_BINARIES_ANALYZERS =~ /\bbundler-audit\b/ - -retire.js: - extends: .download_images - only: - variables: - - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && - $SECURE_BINARIES_ANALYZERS =~ /\bretire\.js\b/ - gemnasium: extends: .download_images only: diff --git a/locale/gitlab.pot b/locale/gitlab.pot index acfa96eacd2..0f19cae937a 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -2595,6 +2595,9 @@ msgstr "" msgid "AdminProjects|Delete Project %{projectName}?" msgstr "" +msgid "AdminSettings|%{strongStart}WARNING:%{strongEnd} Environment variable %{environment_variable} does not exist or is not pointing to a valid directory. %{icon_link}" +msgstr "" + msgid "AdminSettings|A Let's Encrypt account will be configured for this GitLab instance using this email address. You will receive emails to warn of expiring certificates. %{link_start}Learn more.%{link_end}" msgstr "" @@ -2640,6 +2643,9 @@ msgstr "" msgid "AdminSettings|Enable Service Ping" msgstr "" +msgid "AdminSettings|Enable a Prometheus endpoint that exposes health and performance statistics. The Health Check menu item appears in the Monitoring section of the Admin Area. Restart required. %{link_start}Learn more.%{link_end}" +msgstr "" + msgid "AdminSettings|Enable kuromoji custom analyzer: Indexing" msgstr "" @@ -13822,9 +13828,6 @@ msgstr "" msgid "Enable automatic repository housekeeping" msgstr "" -msgid "Enable collection of application metrics. Restart required." -msgstr "" - msgid "Enable container expiration and retention policies for projects created earlier than GitLab 12.7." msgstr "" @@ -14083,9 +14086,6 @@ msgstr "" msgid "Environment scope" msgstr "" -msgid "Environment variable %{environment_variable} does not exist or is not pointing to a valid directory." -msgstr "" - msgid "Environment variables are configured by your administrator to be %{link_start}protected%{link_end} by default." msgstr "" @@ -18769,9 +18769,6 @@ msgstr "" msgid "How the job limiter handles jobs exceeding the thresholds specified below. The 'track' mode only logs the jobs. The 'compress' mode compresses the jobs and raises an exception if the compressed size exceeds the limit." msgstr "" -msgid "How to export these metrics to Prometheus?" -msgstr "" - msgid "I accept the %{terms_link}" msgstr "" @@ -41950,9 +41947,6 @@ msgstr "" msgid "Vulnerability|View training" msgstr "" -msgid "WARNING:" -msgstr "" - msgid "WARNING: This snippet contains hidden files which might be used to mask malicious behavior. Exercise caution if cloning and executing code from this snippet." msgstr "" diff --git a/spec/features/projects/jobs_spec.rb b/spec/features/projects/jobs_spec.rb index b34a615e651..befaf85fc1e 100644 --- a/spec/features/projects/jobs_spec.rb +++ b/spec/features/projects/jobs_spec.rb @@ -484,7 +484,7 @@ RSpec.describe 'Jobs', :clean_gitlab_redis_shared_state do end context 'when job has an initial trace' do - it 'loads job trace' do + it 'loads job logs' do expect(page).to have_content 'BUILD TRACE' job.trace.write(+'a+b') do |stream| diff --git a/spec/frontend/lib/dompurify_spec.js b/spec/frontend/lib/dompurify_spec.js index 47a94a4dcde..34325dad6a1 100644 --- a/spec/frontend/lib/dompurify_spec.js +++ b/spec/frontend/lib/dompurify_spec.js @@ -73,6 +73,16 @@ describe('~/lib/dompurify', () => { expect(sanitize('<p><gl-emoji>💯</gl-emoji></p>')).toBe('<p><gl-emoji>💯</gl-emoji></p>'); }); + it("doesn't allow style tags", () => { + // removes style tags + expect(sanitize('<style>p {width:50%;}</style>')).toBe(''); + expect(sanitize('<style type="text/css">p {width:50%;}</style>')).toBe(''); + // removes mstyle tag (this can removed later by disallowing math tags) + expect(sanitize('<math><mstyle displaystyle="true"></mstyle></math>')).toBe('<math></math>'); + // removes link tag (this is DOMPurify's default behavior) + expect(sanitize('<link rel="stylesheet" href="styles.css">')).toBe(''); + }); + describe.each` type | gon ${'root'} | ${rootGon} diff --git a/spec/frontend/lib/utils/mock_data.js b/spec/frontend/lib/utils/mock_data.js index df1f79529e7..49a2af8b307 100644 --- a/spec/frontend/lib/utils/mock_data.js +++ b/spec/frontend/lib/utils/mock_data.js @@ -3,3 +3,45 @@ export const faviconDataUrl = export const overlayDataUrl = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAA85JREFUWAntVllIVGEUPv/9b46O41KplYN7PeRkti8TjQlhCUGh3MmeQugpIsGKAi2soIcIooiohxYKK2daqDAlIpIiWwxtQaJcaHE0d5tMrbn37z9XRqfR0TvVW56Hudf//uec72zfEWBCJjIwkYGJDPzvGSD/KgExN3Oi2Q+2DJgSDYQEMwItVGH1iZGmJw/Si1y+/PwVAMYYib22MYc/8hVQFgKDEfYoId0KYzagAQebsos/ewMZoeB9wdffcTYpQSaCTWHKoqSQaDk7zkIt0+aCUR8BelEHrf3dUNv9AcqbnsHtT5UKB/hTASh0SLYjnjb/CIDRJi0XiFAaJOpCD8zLpdb4NB66b1OfelthX815dtdRRfiti2aAXLvVLiMQ6olGyztGDkSo4JGGXk8/QFdGpYzpHG2GBQTDhtgVhPEaVbbVpvI6GJz22rv4TcAfrYI1x7Rj5MWWAppomKFVVb2302SFzUkZHAbkG+0b1+Gh77yNYjrmqnWTrLBLRxdvBWv8qlFujH/kYjJYyvLkj71t78zAUvzMAMnHhpN4zf9UREJhd8omyssxu1IgazQDwDnHUcNuH6vhPIE1fmuBzHt74Hn7W89jWGtcAjoaIDOFrdcMYJBkgOCoaRF0Lj0oglddDbCj6tRvKjphEpgjkzEQs2YAKsNxMzjn3nKurhzK+Ly7xe28ua8TwgMMcHJZnvvT0BPtEEKM4tDJ+C8GvIIk4ylINIXVZ0EUKJxYuh3mhCeokbudl6TtVc88dfBdLwbyaWB6zQCYQJpBYSrDGQxBQ/ZWRM2B+VNmQnVnHWx7elyNuL2/R336co7KyJR8CL9oLgEuFlREevWUkEl6uGwpVEG4FBm0OEf9N10NMgPlvWYAuNVwsWDKvcUNYsHUWTCZ13ysyFEXe6TO6aC8CUr9IiK+A05TQrc8yjwmxARHeeMAPlfQJw+AQRwu0YhL/GDXi9NwufG+S8dYkuYMqIb4SsWthotlNMOUCOM6r+G9cqXxPmd1dqrBav/o1zJy2l5/NUjJA/VORwYuFnOUaTQcPs9wMqwV++Xv8oADxKAcZ8nLPr8AoGW+xR6HSqYk3GodAz2QNj0V+Gr26dT9ASNH5239Pf0gktVNWZca8ZvfAFBprWS6hSu1pqt++Y0PD+WIwDAhIWQGtzvSHDbcodfFUFB9hg1Gjs5LXqIdFL+acFBl+FddqYwdxsWC3I70OvgfUaA65zhq2O2c8VxYcyIGFTVlXegYtvCXANCQZJMobjVcLMjtSK/IcEgyOOe8Ve5w7ryKDefp2P3+C/5ohv8HZmVLAAAAAElFTkSuQmCC'; + +const absoluteUrls = [ + 'http://example.org', + 'http://example.org:8080', + 'https://example.org', + 'https://example.org:8080', + 'https://192.168.1.1', +]; + +const rootRelativeUrls = ['/relative/link']; + +const relativeUrls = ['./relative/link', '../relative/link']; + +const urlsWithoutHost = ['http://', 'https://', 'https:https:https:']; + +/* eslint-disable no-script-url */ +const nonHttpUrls = [ + 'javascript:', + 'javascript:alert("XSS")', + 'jav\tascript:alert("XSS");', + '  javascript:alert("XSS");', + 'ftp://192.168.1.1', + 'file:///', + 'file:///etc/hosts', +]; +/* eslint-enable no-script-url */ + +// javascript:alert('XSS') +const encodedJavaScriptUrls = [ + 'javascript:alert('XSS')', + 'javascript:alert('XSS')', + 'javascript:alert('XSS')', + '\\u006A\\u0061\\u0076\\u0061\\u0073\\u0063\\u0072\\u0069\\u0070\\u0074\\u003A\\u0061\\u006C\\u0065\\u0072\\u0074\\u0028\\u0027\\u0058\\u0053\\u0053\\u0027\\u0029', +]; + +export const safeUrls = [...absoluteUrls, ...rootRelativeUrls]; +export const unsafeUrls = [ + ...relativeUrls, + ...urlsWithoutHost, + ...nonHttpUrls, + ...encodedJavaScriptUrls, +]; diff --git a/spec/frontend/lib/utils/url_utility_spec.js b/spec/frontend/lib/utils/url_utility_spec.js index 7608cff4c9e..81cf4bd293b 100644 --- a/spec/frontend/lib/utils/url_utility_spec.js +++ b/spec/frontend/lib/utils/url_utility_spec.js @@ -1,6 +1,7 @@ import setWindowLocation from 'helpers/set_window_location_helper'; import { TEST_HOST } from 'helpers/test_constants'; import * as urlUtils from '~/lib/utils/url_utility'; +import { safeUrls, unsafeUrls } from './mock_data'; const shas = { valid: [ @@ -575,48 +576,6 @@ describe('URL utility', () => { }); describe('isSafeUrl', () => { - const absoluteUrls = [ - 'http://example.org', - 'http://example.org:8080', - 'https://example.org', - 'https://example.org:8080', - 'https://192.168.1.1', - ]; - - const rootRelativeUrls = ['/relative/link']; - - const relativeUrls = ['./relative/link', '../relative/link']; - - const urlsWithoutHost = ['http://', 'https://', 'https:https:https:']; - - /* eslint-disable no-script-url */ - const nonHttpUrls = [ - 'javascript:', - 'javascript:alert("XSS")', - 'jav\tascript:alert("XSS");', - '  javascript:alert("XSS");', - 'ftp://192.168.1.1', - 'file:///', - 'file:///etc/hosts', - ]; - /* eslint-enable no-script-url */ - - // javascript:alert('XSS') - const encodedJavaScriptUrls = [ - 'javascript:alert('XSS')', - 'javascript:alert('XSS')', - 'javascript:alert('XSS')', - '\\u006A\\u0061\\u0076\\u0061\\u0073\\u0063\\u0072\\u0069\\u0070\\u0074\\u003A\\u0061\\u006C\\u0065\\u0072\\u0074\\u0028\\u0027\\u0058\\u0053\\u0053\\u0027\\u0029', - ]; - - const safeUrls = [...absoluteUrls, ...rootRelativeUrls]; - const unsafeUrls = [ - ...relativeUrls, - ...urlsWithoutHost, - ...nonHttpUrls, - ...encodedJavaScriptUrls, - ]; - describe('with URL constructor support', () => { it.each(safeUrls)('returns true for %s', (url) => { expect(urlUtils.isSafeURL(url)).toBe(true); @@ -628,6 +587,16 @@ describe('URL utility', () => { }); }); + describe('sanitizeUrl', () => { + it.each(safeUrls)('returns the url for %s', (url) => { + expect(urlUtils.sanitizeUrl(url)).toBe(url); + }); + + it.each(unsafeUrls)('returns `about:blank` for %s', (url) => { + expect(urlUtils.sanitizeUrl(url)).toBe('about:blank'); + }); + }); + describe('getNormalizedURL', () => { it.each` url | base | result diff --git a/spec/frontend/project_select_combo_button_spec.js b/spec/frontend/project_select_combo_button_spec.js index 40e7d27edc8..1f762d359ea 100644 --- a/spec/frontend/project_select_combo_button_spec.js +++ b/spec/frontend/project_select_combo_button_spec.js @@ -22,6 +22,11 @@ describe('Project Select Combo Button', () => { name: 'My Other Cool Project', url: 'http://myothercoolproject.com', }, + vulnerableProject: { + name: 'Self XSS', + // eslint-disable-next-line no-script-url + url: 'javascript:alert(1)', + }, localStorageKey: 'group-12345-new-issue-recent-project', relativePath: 'issues/new', }; @@ -99,6 +104,25 @@ describe('Project Select Combo Button', () => { }); }); + describe('after selecting a vulnerable project', () => { + beforeEach(() => { + testContext.comboButton = new ProjectSelectComboButton(testContext.projectSelectInput); + + // mock the effect of selecting an item from the projects dropdown (select2) + $('.project-item-select') + .val(JSON.stringify(testContext.defaults.vulnerableProject)) + .trigger('change'); + }); + + it('newItemBtn href is correctly sanitized', () => { + expect(testContext.newItemBtn.getAttribute('href')).toBe('about:blank'); + }); + + afterEach(() => { + window.localStorage.clear(); + }); + }); + describe('deriveTextVariants', () => { beforeEach(() => { testContext.mockExecutionContext = { diff --git a/spec/frontend/vue_shared/security_reports/mock_data.js b/spec/frontend/vue_shared/security_reports/mock_data.js index dac9accbbf5..a9ad675e538 100644 --- a/spec/frontend/vue_shared/security_reports/mock_data.js +++ b/spec/frontend/vue_shared/security_reports/mock_data.js @@ -62,7 +62,7 @@ export const mockFindings = [ report_type: 'dependency_scanning', name: '3rd party CORS request may execute in jquery', severity: 'high', - scanner: { external_id: 'retire.js', name: 'Retire.js' }, + scanner: { external_id: 'gemnasium', name: 'gemnasium' }, identifiers: [ { external_type: 'cve', @@ -145,7 +145,7 @@ export const mockFindings = [ name: 'jQuery before 3.4.0, as used in Drupal, Backdrop CMS, and other products, mishandles jQuery.extend(true, {}, ...) because of Object.prototype pollution in jquery', severity: 'low', - scanner: { external_id: 'retire.js', name: 'Retire.js' }, + scanner: { external_id: 'gemnasium', name: 'gemnasium' }, identifiers: [ { external_type: 'cve', @@ -227,7 +227,7 @@ export const mockFindings = [ name: 'jQuery before 3.4.0, as used in Drupal, Backdrop CMS, and other products, mishandles jQuery.extend(true, {}, ...) because of Object.prototype pollution in jquery', severity: 'low', - scanner: { external_id: 'retire.js', name: 'Retire.js' }, + scanner: { external_id: 'gemnasium', name: 'gemnasium' }, identifiers: [ { external_type: 'cve', diff --git a/spec/lib/container_registry/migration_spec.rb b/spec/lib/container_registry/migration_spec.rb index 654991fd61a..7e8fcd65930 100644 --- a/spec/lib/container_registry/migration_spec.rb +++ b/spec/lib/container_registry/migration_spec.rb @@ -158,6 +158,30 @@ RSpec.describe ContainerRegistry::Migration do end end + describe '.pre_import_timeout' do + let(:value) { 10.minutes } + + before do + stub_application_setting(container_registry_pre_import_timeout: value) + end + + it 'returns the matching application_setting' do + expect(described_class.pre_import_timeout).to eq(value) + end + end + + describe '.import_timeout' do + let(:value) { 10.minutes } + + before do + stub_application_setting(container_registry_import_timeout: value) + end + + it 'returns the matching application_setting' do + expect(described_class.import_timeout).to eq(value) + end + end + describe '.target_plans' do subject { described_class.target_plans } diff --git a/spec/lib/gitlab/ci/reports/security/scanner_spec.rb b/spec/lib/gitlab/ci/reports/security/scanner_spec.rb index eb406e01b24..d7ac82e3b53 100644 --- a/spec/lib/gitlab/ci/reports/security/scanner_spec.rb +++ b/spec/lib/gitlab/ci/reports/security/scanner_spec.rb @@ -103,8 +103,6 @@ RSpec.describe Gitlab::Ci::Reports::Security::Scanner do context 'when the `external_id` of the scanners are different' do where(:scanner_1_attributes, :scanner_2_attributes, :expected_comparison_result) do - { external_id: 'bundler_audit', name: 'foo', vendor: 'bar' } | { external_id: 'retire.js', name: 'foo', vendor: 'bar' } | -1 - { external_id: 'retire.js', name: 'foo', vendor: 'bar' } | { external_id: 'gemnasium', name: 'foo', vendor: 'bar' } | -1 { external_id: 'gemnasium', name: 'foo', vendor: 'bar' } | { external_id: 'gemnasium-maven', name: 'foo', vendor: 'bar' } | -1 { external_id: 'gemnasium-maven', name: 'foo', vendor: 'bar' } | { external_id: 'gemnasium-python', name: 'foo', vendor: 'bar' } | -1 { external_id: 'gemnasium-python', name: 'foo', vendor: 'bar' } | { external_id: 'bandit', name: 'foo', vendor: 'bar' } | 1 diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index 9f0f056d14e..dbced00754a 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -83,10 +83,14 @@ RSpec.describe ApplicationSetting do it { is_expected.to validate_numericality_of(:container_registry_import_max_retries).only_integer.is_greater_than_or_equal_to(0) } it { is_expected.to validate_numericality_of(:container_registry_import_start_max_retries).only_integer.is_greater_than_or_equal_to(0) } it { is_expected.to validate_numericality_of(:container_registry_import_max_step_duration).only_integer.is_greater_than_or_equal_to(0) } + it { is_expected.to validate_numericality_of(:container_registry_pre_import_timeout).only_integer.is_greater_than_or_equal_to(0) } + it { is_expected.to validate_numericality_of(:container_registry_import_timeout).only_integer.is_greater_than_or_equal_to(0) } it { is_expected.not_to allow_value(nil).for(:container_registry_import_max_tags_count) } it { is_expected.not_to allow_value(nil).for(:container_registry_import_max_retries) } it { is_expected.not_to allow_value(nil).for(:container_registry_import_start_max_retries) } it { is_expected.not_to allow_value(nil).for(:container_registry_import_max_step_duration) } + it { is_expected.not_to allow_value(nil).for(:container_registry_pre_import_timeout) } + it { is_expected.not_to allow_value(nil).for(:container_registry_import_timeout) } it { is_expected.to validate_presence_of(:container_registry_import_target_plan) } it { is_expected.to validate_presence_of(:container_registry_import_created_before) } diff --git a/spec/requests/api/ci/jobs_spec.rb b/spec/requests/api/ci/jobs_spec.rb index d3820e4948e..4bd9f81fd1d 100644 --- a/spec/requests/api/ci/jobs_spec.rb +++ b/spec/requests/api/ci/jobs_spec.rb @@ -471,7 +471,7 @@ RSpec.describe API::Ci::Jobs do end context 'authorized user' do - context 'when trace is in ObjectStorage' do + context 'when log is in ObjectStorage' do let!(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) } let(:url) { 'http://object-storage/trace' } let(:file_path) { expand_fixture_path('trace/sample_trace') } @@ -485,49 +485,49 @@ RSpec.describe API::Ci::Jobs do end end - it 'returns specific job trace' do + it 'returns specific job logs' do expect(response).to have_gitlab_http_status(:ok) expect(response.body).to eq(job.trace.raw) end end - context 'when trace is artifact' do + context 'when log is artifact' do let(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) } - it 'returns specific job trace' do + it 'returns specific job log' do expect(response).to have_gitlab_http_status(:ok) expect(response.body).to eq(job.trace.raw) end end - context 'when live trace and uploadless trace artifact' do + context 'when incremental logging and uploadless log artifact' do let(:job) { create(:ci_build, :trace_live, :unarchived_trace_artifact, pipeline: pipeline) } - it 'returns specific job trace' do + it 'returns specific job log' do expect(response).to have_gitlab_http_status(:ok) expect(response.body).to eq(job.trace.raw) end end - context 'when trace is live' do + context 'when log is incremental' do let(:job) { create(:ci_build, :trace_live, pipeline: pipeline) } - it 'returns specific job trace' do + it 'returns specific job log' do expect(response).to have_gitlab_http_status(:ok) expect(response.body).to eq(job.trace.raw) end end - context 'when no trace' do + context 'when no log' do let(:job) { create(:ci_build, pipeline: pipeline) } - it 'returns empty trace' do + it 'returns empty log' do expect(response).to have_gitlab_http_status(:ok) expect(response.body).to be_empty end end - context 'when trace artifact record exists with no stored file' do + context 'when log artifact record exists with no stored file' do let(:job) { create(:ci_build, pipeline: pipeline) } before do @@ -544,7 +544,7 @@ RSpec.describe API::Ci::Jobs do context 'unauthorized user' do let(:api_user) { nil } - it 'does not return specific job trace' do + it 'does not return specific job log' do expect(response).to have_gitlab_http_status(:unauthorized) end end diff --git a/spec/requests/api/ci/runner/jobs_trace_spec.rb b/spec/requests/api/ci/runner/jobs_trace_spec.rb index d6928969beb..c3c074d80d9 100644 --- a/spec/requests/api/ci/runner/jobs_trace_spec.rb +++ b/spec/requests/api/ci/runner/jobs_trace_spec.rb @@ -272,7 +272,7 @@ RSpec.describe API::Ci::Runner, :clean_gitlab_redis_trace_chunks do it { expect(response).to have_gitlab_http_status(:forbidden) } end - context 'when the job trace is too big' do + context 'when the job log is too big' do before do project.actual_limits.update!(ci_jobs_trace_size_limit: 1) end diff --git a/spec/services/merge_requests/merge_service_spec.rb b/spec/services/merge_requests/merge_service_spec.rb index ecb856bd1a4..78deab64b1c 100644 --- a/spec/services/merge_requests/merge_service_spec.rb +++ b/spec/services/merge_requests/merge_service_spec.rb @@ -149,7 +149,7 @@ RSpec.describe MergeRequests::MergeService do allow(project).to receive(:default_branch).and_return(merge_request.target_branch) end - it 'closes GitLab issue tracker issues' do + it 'closes GitLab issue tracker issues', :sidekiq_inline do issue = create :issue, project: project commit = double('commit', safe_message: "Fixes #{issue.to_reference}", date: Time.current, authored_date: Time.current) allow(merge_request).to receive(:commits).and_return([commit]) diff --git a/spec/services/merge_requests/post_merge_service_spec.rb b/spec/services/merge_requests/post_merge_service_spec.rb index 9a7f36929fc..f0885365f96 100644 --- a/spec/services/merge_requests/post_merge_service_spec.rb +++ b/spec/services/merge_requests/post_merge_service_spec.rb @@ -59,21 +59,6 @@ RSpec.describe MergeRequests::PostMergeService do expect(diff_removal_service).to have_received(:execute) end - it 'marks MR as merged regardless of errors when closing issues' do - merge_request.update!(target_branch: 'foo') - allow(project).to receive(:default_branch).and_return('foo') - - issue = create(:issue, project: project) - allow(merge_request).to receive(:visible_closing_issues_for).and_return([issue]) - expect_next_instance_of(Issues::CloseService) do |close_service| - allow(close_service).to receive(:execute).with(issue, commit: merge_request).and_raise(RuntimeError) - end - - expect { subject }.to raise_error(RuntimeError) - - expect(merge_request.reload).to be_merged - end - it 'clean up environments for the merge request' do expect_next_instance_of(::Environments::StopService) do |stop_environment_service| expect(stop_environment_service).to receive(:execute_for_merge_request_pipeline).with(merge_request) @@ -88,6 +73,67 @@ RSpec.describe MergeRequests::PostMergeService do subject end + context 'when there are issues to be closed' do + let_it_be(:issue) { create(:issue, project: project) } + + before do + merge_request.update!(target_branch: 'foo') + + allow(project).to receive(:default_branch).and_return('foo') + allow(merge_request).to receive(:visible_closing_issues_for).and_return([issue]) + end + + it 'performs MergeRequests::CloseIssueWorker asynchronously' do + expect(MergeRequests::CloseIssueWorker) + .to receive(:perform_async) + .with(project.id, user.id, issue.id, merge_request.id) + + subject + + expect(merge_request.reload).to be_merged + end + + context 'when issue is an external issue' do + let_it_be(:issue) { ExternalIssue.new('JIRA-123', project) } + + it 'executes Issues::CloseService' do + expect_next_instance_of(Issues::CloseService) do |close_service| + expect(close_service).to receive(:execute).with(issue, commit: merge_request) + end + + subject + + expect(merge_request.reload).to be_merged + end + end + + context 'when async_mr_close_issue feature flag is disabled' do + before do + stub_feature_flags(async_mr_close_issue: false) + end + + it 'executes Issues::CloseService' do + expect_next_instance_of(Issues::CloseService) do |close_service| + expect(close_service).to receive(:execute).with(issue, commit: merge_request) + end + + subject + + expect(merge_request.reload).to be_merged + end + + it 'marks MR as merged regardless of errors when closing issues' do + expect_next_instance_of(Issues::CloseService) do |close_service| + allow(close_service).to receive(:execute).with(issue, commit: merge_request).and_raise(RuntimeError) + end + + expect { subject }.to raise_error(RuntimeError) + + expect(merge_request.reload).to be_merged + end + end + end + context 'when the merge request has review apps' do it 'cancels all review app deployments' do pipeline = create(:ci_pipeline, diff --git a/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb b/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb index 25a3723fbaa..1c8a1c6a171 100644 --- a/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb +++ b/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb @@ -17,7 +17,7 @@ RSpec.describe 'gitlab:artifacts namespace rake task', :silence_stdout do subject { run_rake_task('gitlab:artifacts:migrate') } let!(:artifact) { create(:ci_job_artifact, :archive, file_store: store) } - let!(:job_trace) { create(:ci_job_artifact, :trace, file_store: store) } + let!(:job_log) { create(:ci_job_artifact, :trace, file_store: store) } context 'when local storage is used' do let(:store) { ObjectStorage::Store::LOCAL } @@ -29,7 +29,7 @@ RSpec.describe 'gitlab:artifacts namespace rake task', :silence_stdout do subject expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE) - expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE) + expect(job_log.reload.file_store).to eq(ObjectStorage::Store::REMOTE) end end @@ -38,7 +38,7 @@ RSpec.describe 'gitlab:artifacts namespace rake task', :silence_stdout do subject expect(artifact.reload.file_store).to eq(ObjectStorage::Store::LOCAL) - expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::LOCAL) + expect(job_log.reload.file_store).to eq(ObjectStorage::Store::LOCAL) end end end @@ -51,7 +51,7 @@ RSpec.describe 'gitlab:artifacts namespace rake task', :silence_stdout do subject expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE) - expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE) + expect(job_log.reload.file_store).to eq(ObjectStorage::Store::REMOTE) end end end @@ -62,7 +62,7 @@ RSpec.describe 'gitlab:artifacts namespace rake task', :silence_stdout do subject { run_rake_task('gitlab:artifacts:migrate_to_local') } let!(:artifact) { create(:ci_job_artifact, :archive, file_store: store) } - let!(:job_trace) { create(:ci_job_artifact, :trace, file_store: store) } + let!(:job_log) { create(:ci_job_artifact, :trace, file_store: store) } context 'when remote storage is used' do let(:store) { ObjectStorage::Store::REMOTE } @@ -72,7 +72,7 @@ RSpec.describe 'gitlab:artifacts namespace rake task', :silence_stdout do subject expect(artifact.reload.file_store).to eq(ObjectStorage::Store::LOCAL) - expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::LOCAL) + expect(job_log.reload.file_store).to eq(ObjectStorage::Store::LOCAL) end end end @@ -84,7 +84,7 @@ RSpec.describe 'gitlab:artifacts namespace rake task', :silence_stdout do subject expect(artifact.reload.file_store).to eq(ObjectStorage::Store::LOCAL) - expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::LOCAL) + expect(job_log.reload.file_store).to eq(ObjectStorage::Store::LOCAL) end end end diff --git a/spec/workers/container_registry/migration/guard_worker_spec.rb b/spec/workers/container_registry/migration/guard_worker_spec.rb index 402f8989fab..0fbc648e77f 100644 --- a/spec/workers/container_registry/migration/guard_worker_spec.rb +++ b/spec/workers/container_registry/migration/guard_worker_spec.rb @@ -25,7 +25,7 @@ RSpec.describe ContainerRegistry::Migration::GuardWorker, :aggregate_failures do allow(::Gitlab).to receive(:com?).and_return(true) end - shared_examples 'handling long running migrations' do + shared_examples 'handling long running migrations' do |timeout:| before do allow_next_found_instance_of(ContainerRepository) do |repository| allow(repository).to receive(:migration_cancel).and_return(migration_cancel_response) @@ -37,12 +37,26 @@ RSpec.describe ContainerRegistry::Migration::GuardWorker, :aggregate_failures do expect(worker).to receive(:log_extra_metadata_on_done).with(:stale_migrations_count, 1) expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_stale_migrations_count, 1) expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_long_running_migration_ids, [stale_migration.id]) + expect(ContainerRegistry::Migration).to receive(timeout).and_call_original expect { subject } .to change(import_aborted_migrations, :count).by(1) .and change { stale_migration.reload.migration_state }.to('import_aborted') .and not_change { ongoing_migration.migration_state } end + + context 'registry_migration_guard_thresholds feature flag disabled' do + before do + stub_feature_flags(registry_migration_guard_thresholds: false) + end + + it 'falls back on the hardcoded value' do + expect(ContainerRegistry::Migration).not_to receive(:pre_import_timeout) + + expect { subject } + .to change { stale_migration.reload.migration_state }.to('import_aborted') + end + end end context 'migration is canceled' do @@ -61,6 +75,7 @@ RSpec.describe ContainerRegistry::Migration::GuardWorker, :aggregate_failures do expect(worker).to receive(:log_extra_metadata_on_done).with(:stale_migrations_count, 1) expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_stale_migrations_count, 1) expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_long_running_migration_ids, [stale_migration.id]) + expect(ContainerRegistry::Migration).to receive(timeout).and_call_original expect { subject } .to change(import_skipped_migrations, :count) @@ -68,6 +83,19 @@ RSpec.describe ContainerRegistry::Migration::GuardWorker, :aggregate_failures do expect(stale_migration.reload.migration_state).to eq('import_skipped') expect(stale_migration.reload.migration_skipped_reason).to eq('migration_canceled') end + + context 'registry_migration_guard_thresholds feature flag disabled' do + before do + stub_feature_flags(registry_migration_guard_thresholds: false) + end + + it 'falls back on the hardcoded value' do + expect(ContainerRegistry::Migration).not_to receive(timeout) + + expect { subject } + .to change { stale_migration.reload.migration_state }.to('import_skipped') + end + end end context 'when the retry limit has not been reached' do @@ -112,6 +140,8 @@ RSpec.describe ContainerRegistry::Migration::GuardWorker, :aggregate_failures do allow_next_instance_of(ContainerRegistry::GitlabApiClient) do |client| allow(client).to receive(:import_status).and_return(import_status) end + + stub_application_setting(container_registry_pre_import_timeout: 10.minutes) end it 'will abort the migration' do @@ -131,7 +161,7 @@ RSpec.describe ContainerRegistry::Migration::GuardWorker, :aggregate_failures do context 'the client returns pre_import_in_progress' do let(:import_status) { 'pre_import_in_progress' } - it_behaves_like 'handling long running migrations' + it_behaves_like 'handling long running migrations', timeout: :pre_import_timeout end end @@ -167,6 +197,8 @@ RSpec.describe ContainerRegistry::Migration::GuardWorker, :aggregate_failures do allow_next_instance_of(ContainerRegistry::GitlabApiClient) do |client| allow(client).to receive(:import_status).and_return(import_status) end + + stub_application_setting(container_registry_import_timeout: 10.minutes) end it 'will abort the migration' do @@ -186,7 +218,7 @@ RSpec.describe ContainerRegistry::Migration::GuardWorker, :aggregate_failures do context 'the client returns import_in_progress' do let(:import_status) { 'import_in_progress' } - it_behaves_like 'handling long running migrations' + it_behaves_like 'handling long running migrations', timeout: :import_timeout end end end diff --git a/spec/workers/merge_requests/close_issue_worker_spec.rb b/spec/workers/merge_requests/close_issue_worker_spec.rb new file mode 100644 index 00000000000..5e6bdc2a43e --- /dev/null +++ b/spec/workers/merge_requests/close_issue_worker_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe MergeRequests::CloseIssueWorker do + subject(:worker) { described_class.new } + + describe '#perform' do + let!(:user) { create(:user) } + let!(:project) { create(:project) } + let!(:issue) { create(:issue, project: project) } + let!(:merge_request) { create(:merge_request, source_project: project) } + + it 'calls the close issue service' do + expect_next_instance_of(Issues::CloseService, project: project, current_user: user) do |service| + expect(service).to receive(:execute).with(issue, commit: merge_request) + end + + subject.perform(project.id, user.id, issue.id, merge_request.id) + end + + shared_examples 'when object does not exist' do + it 'does not call the close issue service' do + expect(Issues::CloseService).not_to receive(:new) + + expect { subject.perform(project.id, user.id, issue.id, merge_request.id) } + .not_to raise_exception + end + end + + context 'when the project does not exist' do + before do + project.destroy! + end + + it_behaves_like 'when object does not exist' + end + + context 'when the user does not exist' do + before do + user.destroy! + end + + it_behaves_like 'when object does not exist' + end + + context 'when the issue does not exist' do + before do + issue.destroy! + end + + it_behaves_like 'when object does not exist' + end + + context 'when the merge request does not exist' do + before do + merge_request.destroy! + end + + it_behaves_like 'when object does not exist' + end + end +end |