diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-04-08 18:09:16 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-04-08 18:09:16 +0000 |
| commit | 80e9fdc9682cfbcfb9202a2733605a6a6bd23f05 (patch) | |
| tree | 168375ea13d1a1f01d4cbcf6f0513fc9883e9477 | |
| parent | 5372e109c0660e4670aa987568a51082beca1b3c (diff) | |
| download | gitlab-ce-80e9fdc9682cfbcfb9202a2733605a6a6bd23f05.tar.gz | |
Add latest changes from gitlab-org/gitlab@master
37 files changed, 246 insertions, 61 deletions
diff --git a/.gitlab/issue_templates/Migrations.md b/.gitlab/issue_templates/Migrations.md index 378e3b01f6a..38fc7a31b24 100644 --- a/.gitlab/issue_templates/Migrations.md +++ b/.gitlab/issue_templates/Migrations.md @@ -63,5 +63,5 @@ Here is an example of what this list might look like once populated: https://git ------ -/label ~Open-Source ~movingtogitlab +/label ~"Open Source" ~movingtogitlab /cc @nuritzi
\ No newline at end of file diff --git a/app/controllers/groups/settings/integrations_controller.rb b/app/controllers/groups/settings/integrations_controller.rb index 0ba030f26bc..adfbe9bfa17 100644 --- a/app/controllers/groups/settings/integrations_controller.rb +++ b/app/controllers/groups/settings/integrations_controller.rb @@ -7,6 +7,10 @@ module Groups before_action :authorize_admin_group! + def index + @integrations = [] + end + private # TODO: Make this compatible with group-level integration diff --git a/app/helpers/groups_helper.rb b/app/helpers/groups_helper.rb index 5abe6a635e0..0d7e2a7bd38 100644 --- a/app/helpers/groups_helper.rb +++ b/app/helpers/groups_helper.rb @@ -11,7 +11,18 @@ module GroupsHelper end def group_nav_link_paths - %w[groups#projects groups#edit badges#index ci_cd#show ldap_group_links#index hooks#index audit_events#index pipeline_quota#index] + %w[ + groups#projects + groups#edit + badges#index + ci_cd#show + integrations#index + integrations#edit + ldap_group_links#index + hooks#index + audit_events#index + pipeline_quota#index + ] end def group_packages_nav_link_paths diff --git a/app/helpers/services_helper.rb b/app/helpers/services_helper.rb index fe2df918819..a4b6e613100 100644 --- a/app/helpers/services_helper.rb +++ b/app/helpers/services_helper.rb @@ -62,10 +62,6 @@ module ServicesHelper !current_controller?("admin/services") && service.deprecated? end - def edit_integration_path(integration) - edit_admin_application_settings_integration_path(integration) - end - def scoped_integrations_path if @project.present? project_settings_integrations_path(@project) @@ -86,6 +82,16 @@ module ServicesHelper end end + def scoped_edit_integration_path(integration) + if @project.present? + edit_project_settings_integration_path(@project, integration) + elsif @group.present? + edit_group_settings_integration_path(@group, integration) + else + edit_admin_application_settings_integration_path(integration) + end + end + def scoped_test_integration_path(integration) if @project.present? test_project_settings_integration_path(@project, integration) diff --git a/app/models/project_import_state.rb b/app/models/project_import_state.rb index 52fd23aefd5..f58b8dc624d 100644 --- a/app/models/project_import_state.rb +++ b/app/models/project_import_state.rb @@ -10,6 +10,8 @@ class ProjectImportState < ApplicationRecord validates :project, presence: true + alias_attribute :correlation_id, :correlation_id_value + state_machine :status, initial: :none do event :schedule do transition [:none, :finished, :failed] => :scheduled @@ -39,7 +41,11 @@ class ProjectImportState < ApplicationRecord after_transition [:none, :finished, :failed] => :scheduled do |state, _| state.run_after_commit do job_id = project.add_import_job - update(jid: job_id) if job_id + + if job_id + correlation_id = Labkit::Correlation::CorrelationId.current_or_new_id + update(jid: job_id, correlation_id_value: correlation_id) + end end end diff --git a/app/serializers/diffs_entity.rb b/app/serializers/diffs_entity.rb index 1d0b4183f96..568d0f6aa8f 100644 --- a/app/serializers/diffs_entity.rb +++ b/app/serializers/diffs_entity.rb @@ -99,7 +99,7 @@ class DiffsEntity < Grape::Entity end def commit_options(options) - prev_commit_id, next_commit_id = *commit_neighbors(options[:commit]&.id) + next_commit_id, prev_commit_id = *commit_neighbors(options[:commit]&.id) options.merge( type: :full, diff --git a/app/views/admin/application_settings/integrations.html.haml b/app/views/admin/application_settings/integrations.html.haml index 14df0599583..2b01160a230 100644 --- a/app/views/admin/application_settings/integrations.html.haml +++ b/app/views/admin/application_settings/integrations.html.haml @@ -18,7 +18,7 @@ %p = s_('AdminSettings|Integrations configured here will automatically apply to all projects on this instance.') = link_to _('Learn more'), '#' - = render 'projects/services/integrations' + = render 'shared/integrations/integrations', integrations: @integrations - else = render_if_exists 'admin/application_settings/elasticsearch_form' diff --git a/app/views/groups/settings/integrations/index.html.haml b/app/views/groups/settings/integrations/index.html.haml new file mode 100644 index 00000000000..78825cc72b0 --- /dev/null +++ b/app/views/groups/settings/integrations/index.html.haml @@ -0,0 +1,9 @@ +- breadcrumb_title _('Integrations') +- page_title _('Integrations') +- @content_class = 'limit-container-width' unless fluid_layout + +%h4= s_('GroupSettings|Apply integration settings to all Projects') +%p + = s_('GroupSettings|Integrations configured here will automatically apply to all projects in this group.') + = link_to _('Learn more'), '#' += render 'shared/integrations/integrations', integrations: @integrations diff --git a/app/views/layouts/nav/sidebar/_group.html.haml b/app/views/layouts/nav/sidebar/_group.html.haml index b3acf3320d3..89bcccb6185 100644 --- a/app/views/layouts/nav/sidebar/_group.html.haml +++ b/app/views/layouts/nav/sidebar/_group.html.haml @@ -159,6 +159,11 @@ = link_to group_settings_ci_cd_path(@group), title: _('CI / CD') do %span = _('CI / CD') + - if Feature.enabled?(:group_level_integrations, @group) + = nav_link(controller: :integrations) do + = link_to group_settings_integrations_path(@group), title: _('Integrations') do + %span + = _('Integrations') = render_if_exists "groups/ee/settings_nav" diff --git a/app/views/projects/services/_integrations.html.haml b/app/views/shared/integrations/_integrations.html.haml index 24aeb598d7b..b2359aca016 100644 --- a/app/views/projects/services/_integrations.html.haml +++ b/app/views/shared/integrations/_integrations.html.haml @@ -12,12 +12,12 @@ %th{ role: 'columnheader', scope: 'col', 'aria-colindex': 4 }= _('Last updated') %tbody{ role: 'rowgroup' } - - @integrations.each do |integration| + - integrations.each do |integration| %tr{ role: 'row' } %td{ role: 'cell', 'aria-colindex': 1 } = boolean_to_icon integration.activated? %td{ role: 'cell', 'aria-colindex': 2 } - = link_to edit_integration_path(integration) do + = link_to scoped_edit_integration_path(integration) do %strong= integration.title %td.d-none.d-sm-block{ role: 'cell', 'aria-colindex': 3 } = integration.description diff --git a/changelogs/unreleased/210522-import-status-correlation-id.yml b/changelogs/unreleased/210522-import-status-correlation-id.yml new file mode 100644 index 00000000000..06491a1c6ec --- /dev/null +++ b/changelogs/unreleased/210522-import-status-correlation-id.yml @@ -0,0 +1,5 @@ +--- +title: Add correlation_id to project_mirror_data, expose in /import API endpoints +merge_request: 28662 +author: +type: changed diff --git a/config/initializers/sidekiq_cluster.rb b/config/initializers/sidekiq_cluster.rb index 4622984fe0c..4ff8dd9b936 100644 --- a/config/initializers/sidekiq_cluster.rb +++ b/config/initializers/sidekiq_cluster.rb @@ -13,6 +13,16 @@ if ENV['ENABLE_SIDEKIQ_CLUSTER'] # this case the parent PID changes and we need to terminate ourselves. if Process.ppid != parent Process.kill(:TERM, Process.pid) + + # Wait for just a few extra seconds for a final attempt to + # gracefully terminate. Considering the parent (cluster) process + # have changed (SIGKILL'd), it shouldn't take long to shutdown. + sleep(5) + + # Signaling the Sidekiq Pgroup as KILL is not forwarded to + # a possible child process. In Sidekiq Cluster, all child Sidekiq + # processes are PGROUP leaders (each process has its own pgroup). + Process.kill(:KILL, 0) break end end diff --git a/db/migrate/20200402124802_add_correlation_id_to_project_import_state.rb b/db/migrate/20200402124802_add_correlation_id_to_project_import_state.rb new file mode 100644 index 00000000000..974009b101a --- /dev/null +++ b/db/migrate/20200402124802_add_correlation_id_to_project_import_state.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddCorrelationIdToProjectImportState < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + with_lock_retries do + add_column :project_mirror_data, :correlation_id_value, :string, limit: 128 + end + end + + def down + with_lock_retries do + remove_column :project_mirror_data, :correlation_id_value + end + end +end diff --git a/db/structure.sql b/db/structure.sql index c2a2e4b361c..459d04570cc 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -4932,7 +4932,8 @@ CREATE TABLE public.project_mirror_data ( jid character varying, last_error text, last_update_at timestamp with time zone, - last_successful_update_at timestamp with time zone + last_successful_update_at timestamp with time zone, + correlation_id_value character varying(128) ); CREATE SEQUENCE public.project_mirror_data_id_seq @@ -13062,6 +13063,7 @@ COPY "schema_migrations" (version) FROM STDIN; 20200331220930 20200401211005 20200402123926 +20200402124802 20200402135250 20200402185044 20200403184110 diff --git a/doc/api/epic_links.md b/doc/api/epic_links.md index 36503942729..4f45fbde9e7 100644 --- a/doc/api/epic_links.md +++ b/doc/api/epic_links.md @@ -80,7 +80,7 @@ POST /groups/:id/epics/:epic_iid/epics | `child_epic_id` | integer | yes | The global ID of the child epic. Internal ID can't be used because they can conflict with epics from other groups. | ```shell -curl --header POST "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5/epics/6 +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5/epics/6 ``` Example response: @@ -133,7 +133,7 @@ POST /groups/:id/epics/:epic_iid/epics | `title` | string | yes | The title of a newly created epic. | ```shell -curl --header POST "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5/epics?title=Newpic +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5/epics?title=Newpic ``` Example response: @@ -168,7 +168,7 @@ PUT /groups/:id/epics/:epic_iid/epics/:child_epic_id | `move_after_id` | integer | no | The global ID of a sibling epic that should be placed after the child epic. | ```shell -curl --header PUT "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/4/epics/5 +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/4/epics/5 ``` Example response: @@ -223,7 +223,7 @@ DELETE /groups/:id/epics/:epic_iid/epics/:child_epic_id | `child_epic_id` | integer | yes | The global ID of the child epic. Internal ID can't be used because they can conflict with epics from other groups. | ```shell -curl --header DELETE "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/4/epics/5 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/4/epics/5 ``` Example response: diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index 8ad2e84a03e..78022e8e754 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -172,7 +172,8 @@ requests.post(url, headers=headers, data=data, files=files) "path": "api-project", "path_with_namespace": "root/api-project", "created_at": "2018-02-13T09:05:58.023Z", - "import_status": "scheduled" + "import_status": "scheduled", + "correlation_id": "mezklWso3Za" } ``` @@ -211,6 +212,7 @@ If the status is `failed`, it will include the import error message under `impor "path": "gitlab-test", "path_with_namespace": "gitlab-org/gitlab-test", "created_at": "2017-08-29T04:36:44.383Z", - "import_status": "started" + "import_status": "started", + "correlation_id": "mezklWso3Za" } ``` diff --git a/doc/development/foreign_keys.md b/doc/development/foreign_keys.md index 38b60ce6f0b..508e5665f08 100644 --- a/doc/development/foreign_keys.md +++ b/doc/development/foreign_keys.md @@ -87,3 +87,13 @@ create_table :user_configs, id: false do |t| ... end ``` + +You will also need to add the new primary key to the model: + +```ruby +class UserConfig < ActiveRecord::Base + self.primary_key = :user_id + + belongs_to :user +end +``` diff --git a/doc/topics/web_application_firewall/img/guide_waf_ingress_installation_v12_10.png b/doc/topics/web_application_firewall/img/guide_waf_ingress_installation_v12_10.png Binary files differnew file mode 100644 index 00000000000..e88f62a2eba --- /dev/null +++ b/doc/topics/web_application_firewall/img/guide_waf_ingress_installation_v12_10.png diff --git a/doc/topics/web_application_firewall/img/guide_waf_ingress_installation_v12_9.png b/doc/topics/web_application_firewall/img/guide_waf_ingress_installation_v12_9.png Binary files differdeleted file mode 100644 index 5c4718b0487..00000000000 --- a/doc/topics/web_application_firewall/img/guide_waf_ingress_installation_v12_9.png +++ /dev/null diff --git a/doc/topics/web_application_firewall/img/guide_waf_ingress_save_changes_v12_10.png b/doc/topics/web_application_firewall/img/guide_waf_ingress_save_changes_v12_10.png Binary files differnew file mode 100644 index 00000000000..1c99d4f7f96 --- /dev/null +++ b/doc/topics/web_application_firewall/img/guide_waf_ingress_save_changes_v12_10.png diff --git a/doc/topics/web_application_firewall/img/guide_waf_ingress_save_changes_v12_9.png b/doc/topics/web_application_firewall/img/guide_waf_ingress_save_changes_v12_9.png Binary files differdeleted file mode 100644 index df1223f12ec..00000000000 --- a/doc/topics/web_application_firewall/img/guide_waf_ingress_save_changes_v12_9.png +++ /dev/null diff --git a/doc/topics/web_application_firewall/quick_start_guide.md b/doc/topics/web_application_firewall/quick_start_guide.md index 768293a6153..1d3380230af 100644 --- a/doc/topics/web_application_firewall/quick_start_guide.md +++ b/doc/topics/web_application_firewall/quick_start_guide.md @@ -104,7 +104,10 @@ For this guide, we need to install Ingress. Ingress provides load balancing, SSL termination, and name-based virtual hosting, using NGINX behind the scenes. Make sure to switch the toogle to the enabled position before installing. - +Both logging and blocking modes are available for WAF. While logging mode is useful for +auditing anomalous traffic, blocking mode ensures the traffic doesn't reach past Ingress. + + After Ingress is installed, wait a few seconds and copy the IP address that is displayed in order to add in your base **Domain** at the top of the page. For diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index a178751da7d..7032ec8c75e 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -16,7 +16,7 @@ If you are using [GitLab CI/CD](../../../ci/README.md), you can analyze your dep vulnerabilities using Dependency Scanning. All dependencies are scanned, including the transitive dependencies (also known as nested dependencies). -You can take advantage of Dependency Scanning by either [including the CI job](#configuration) +You can take advantage of Dependency Scanning by either [including the Dependency Scanning template](#configuration) in your existing `.gitlab-ci.yml` file or by implicitly using [Auto Dependency Scanning](../../../topics/autodevops/stages.md#auto-dependency-scanning-ultimate) that is provided by [Auto DevOps](../../../topics/autodevops/index.md). @@ -137,19 +137,26 @@ using environment variables. The following variables allow configuration of global dependency scanning settings. +| Environment variable | Description | +| --------------------------------------- |------------ | +| `DS_ANALYZER_IMAGE_PREFIX` | Override the name of the Docker registry providing the official default images (proxy). Read more about [customizing analyzers](analyzers.md). | +| `DS_DEFAULT_ANALYZERS` | Override the names of the official default images. Read more about [customizing analyzers](analyzers.md). | +| `DS_DISABLE_DIND` | Disable Docker-in-Docker and run analyzers [individually](#disabling-docker-in-docker-for-dependency-scanning).| +| `ADDITIONAL_CA_CERT_BUNDLE` | Bundle of CA certs to trust. | +| `DS_EXCLUDED_PATHS` | Exclude vulnerabilities from output based on the paths. A comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec`). Parent directories also match patterns. | + +#### Configuring Docker-in-Docker orchestrator + +The following variables configure the Docker-in-Docker orchestrator. + | Environment variable | Default | Description | | --------------------------------------- | ----------- | ----------- | | `DS_ANALYZER_IMAGES` | | Comma separated list of custom images. The official default images are still enabled. Read more about [customizing analyzers](analyzers.md). | -| `DS_ANALYZER_IMAGE_PREFIX` | | Override the name of the Docker registry providing the official default images (proxy). Read more about [customizing analyzers](analyzers.md). | | `DS_ANALYZER_IMAGE_TAG` | | Override the Docker tag of the official default images. Read more about [customizing analyzers](analyzers.md). | -| `DS_DEFAULT_ANALYZERS` | | Override the names of the official default images. Read more about [customizing analyzers](analyzers.md). | -| `DS_DISABLE_DIND` | | Disable Docker in Docker and run analyzers [individually](#disabling-docker-in-docker-for-dependency-scanning).| | `DS_PULL_ANALYZER_IMAGES` | | Pull the images from the Docker registry (set to `0` to disable). | -| `DS_EXCLUDED_PATHS` | | Exclude vulnerabilities from output based on the paths. A comma-separated list of patterns. Patterns can be globs, file or folder paths (for example, `doc,spec`). Parent directories will also match patterns. | | `DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT` | 2m | Time limit for Docker client negotiation. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, or `h`. For example, `300ms`, `1.5h`, or `2h45m`. | | `DS_PULL_ANALYZER_IMAGE_TIMEOUT` | 5m | Time limit when pulling an analyzer's image. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, or `h`. For example, `300ms`, `1.5h`, or `2h45m`. | | `DS_RUN_ANALYZER_TIMEOUT` | 20m | Time limit when running an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, or `h`. For example, `300ms`, `1.5h`, or `2h45m`. | -| `ADDITIONAL_CA_CERT_BUNDLE` | | Bundle of CA certs that you want to trust. | #### Configuring specific analyzers used by Dependency Scanning @@ -204,7 +211,11 @@ to start relevant analyzers depending on the detected repository language(s) ins are some differences in the way repository languages are detected between DIND and non-DIND. You can observe these differences by checking both Linguist and the common library. For instance, Linguist looks for `*.java` files to spin up the [gemnasium-maven](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven) -image, while orchestrator only looks for the existence of `pom.xml` or `build.gradle`. +image, while orchestrator only looks for the existence of `pom.xml` or `build.gradle`. GitLab uses +Linguist to detect new file types in the default branch. This means that when introducing files or +dependencies for a new language or package manager, the corresponding scans won't be triggered in +the MR and will only run on the default branch once the MR is merged. This will be addressed by +[#211702](https://gitlab.com/gitlab-org/gitlab/-/issues/211702). ## Interacting with the vulnerabilities diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index 9cd2e9d4e62..3f172285014 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -18,7 +18,7 @@ vulnerabilities using Static Application Security Testing (SAST). You can take advantage of SAST by doing one of the following: -- [Including the CI job](#configuration) in your existing `.gitlab-ci.yml` file. +- [Including the SAST template](#configuration) in your existing `.gitlab-ci.yml` file. - Implicitly using [Auto SAST](../../../topics/autodevops/stages.md#auto-sast-ultimate) provided by [Auto DevOps](../../../topics/autodevops/index.md). @@ -193,7 +193,15 @@ are some differences in the way repository languages are detected between DIND a observe these differences by checking both Linguist and the common library. For instance, Linguist looks for `*.java` files to spin up the [spotbugs](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) image, while orchestrator only looks for the existence of `pom.xml`, `build.xml`, `gradlew`, -`grailsw`, or `mvnw`. +`grailsw`, or `mvnw`. GitLab uses Linguist to detect new file types in the default branch. This +means that when introducing files or dependencies for a new language or package manager, the +corresponding scans won't be triggered in the MR and will only run on the default branch once the +MR is merged. This will be addressed by [#211702](https://gitlab.com/gitlab-org/gitlab/-/issues/211702). + +NOTE: **Note:** +With the current language detection logic, any new languages or frameworks introduced within the +context of a merge request don't trigger a corresponding scan. These scans only occur once the code +is committed to the default branch. #### Enabling kubesec analyzer @@ -279,12 +287,10 @@ The following are Docker image-related variables. | Environment variable | Description | |------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `SAST_ANALYZER_IMAGES` | Comma separated list of custom images. Default images are still enabled. Read more about [customizing analyzers](analyzers.md). Not available when [Docker in Docker is disabled](#disabling-docker-in-docker-for-sast). | | `SAST_ANALYZER_IMAGE_PREFIX` | Override the name of the Docker registry providing the default images (proxy). Read more about [customizing analyzers](analyzers.md). | -| `SAST_ANALYZER_IMAGE_TAG` | Override the Docker tag of the default images. Read more about [customizing analyzers](analyzers.md). | +| `SAST_ANALYZER_IMAGE_TAG` | **DEPRECATED:** Override the Docker tag of the default images. Read more about [customizing analyzers](analyzers.md). | | `SAST_DEFAULT_ANALYZERS` | Override the names of default images. Read more about [customizing analyzers](analyzers.md). | | `SAST_DISABLE_DIND` | Disable Docker in Docker and run analyzers [individually](#disabling-docker-in-docker-for-sast). | -| `SAST_PULL_ANALYZER_IMAGES` | Pull the images from the Docker registry (set to 0 to disable). Read more about [customizing analyzers](analyzers.md). Not available when [Docker in Docker is disabled](#disabling-docker-in-docker-for-sast). | #### Vulnerability filters @@ -302,12 +308,14 @@ Some analyzers make it possible to filter out vulnerabilities under a given thre | `SAST_GITLEAKS_COMMIT_TO` | - | The commit a gitleaks scan ends at. | | `SAST_GITLEAKS_HISTORIC_SCAN` | false | Flag to enable a historic gitleaks scan. | -#### Timeouts +#### Docker-in-Docker orchestrator -The following variables configure timeouts. +The following variables configure the Docker-in-Docker orchestrator. -| Environment variable | Default value | Description | -|----------------------|---------------|-------------| +| Environment variable | Default value | Description | +|------------------------------------------|---------------|-------------| +| `SAST_ANALYZER_IMAGES` | | Comma-separated list of custom images. Default images are still enabled. Read more about [customizing analyzers](analyzers.md). Not available when [Docker-in-Docker is disabled](#disabling-docker-in-docker-for-sast). | +| `SAST_PULL_ANALYZER_IMAGES` | 1 | Pull the images from the Docker registry (set to 0 to disable). Read more about [customizing analyzers](analyzers.md). Not available when [Docker-in-Docker is disabled](#disabling-docker-in-docker-for-sast). | | `SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT` | 2m | Time limit for Docker client negotiation. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". For example, "300ms", "1.5h" or "2h45m". | | `SAST_PULL_ANALYZER_IMAGE_TIMEOUT` | 5m | Time limit when pulling the image of an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". For example, "300ms", "1.5h" or "2h45m". | | `SAST_RUN_ANALYZER_TIMEOUT` | 20m | Time limit when running an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". For example, "300ms", "1.5h" or "2h45m".| diff --git a/doc/user/application_security/security_dashboard/index.md b/doc/user/application_security/security_dashboard/index.md index 93a325e6777..1eef6b9b696 100644 --- a/doc/user/application_security/security_dashboard/index.md +++ b/doc/user/application_security/security_dashboard/index.md @@ -49,7 +49,7 @@ Visit the page for any pipeline which has run any of the [supported reports](#su > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/6165) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.1. At the project level, the Security Dashboard displays the latest security reports -for your project. Use it to find and fix vulnerabilities affecting the +for your project from the last successful pipeline. Use it to find and fix vulnerabilities affecting the [default branch](../../project/repository/branches/index.md#default-branch).  @@ -74,7 +74,8 @@ Once you're on the dashboard, at the top you should see a series of filters for: To the right of the filters, you should see a **Hide dismissed** toggle button. NOTE: **Note:** -The dashboard only shows projects with [security reports](#supported-reports) enabled in a group. +The dashboard only shows projects with [security reports](#supported-reports) enabled in a group +according to the last successful projects' pipelines.  diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index 7cedd21edcb..ab2aad3b043 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -295,7 +295,7 @@ from processing any requests for the given application or environment. 1. Switching its respective toggle to the disabled position and applying changes through the **Save changes** button. This will reinstall Ingress with the recent changes. - + ##### Viewing Web Application Firewall traffic diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index 4a90cc0a72f..96b051c1673 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -250,6 +250,28 @@ license_scanning: license_scanning: gl-license-scanning-report.json ``` +## Running License Compliance in an offline environment + +License Compliance can be executed on an offline GitLab Ultimate installation by using the following +process: + +1. Host the License Compliance image + `registry.gitlab.com/gitlab-org/security-products/license-management:latest` in your local Docker + container registry. +1. Add the following configuration to your `.gitlab-ci.yml` file. You must replace `image` to refer + to the License Compliance Docker image hosted on your local Docker container registry: + + ```yaml + include: + - template: License-Scanning.gitlab-ci.yml + + license_scanning: + image: registry.example.com/namespace/license-management:latest + ``` + +1. Ensure the package registry is reachable from within the GitLab environment and that the package + manager is configured to use your preferred package registry. + ## Project policies for License Compliance > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5940) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.4. diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index 2f4f3dd5392..5bc71337e44 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -206,7 +206,7 @@ If you don't have editing permission in a board, you're still able to see the co ### Focus mode > - Introduced in [GitLab Starter 9.1](https://about.gitlab.com/releases/2017/04/22/gitlab-9-1-released/#issue-boards-focus-mode-ees-eep). -> - Focus mode moved to GitLab Core in GitLab 12.10. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212331) to GitLab Core in 12.10. Click the button at the top right to toggle focus mode on and off. In focus mode, the navigation UI is hidden, allowing you to focus on issues in the board. diff --git a/lib/api/entities/project_import_status.rb b/lib/api/entities/project_import_status.rb index 0b884b43e9e..9b7a2bd1f3e 100644 --- a/lib/api/entities/project_import_status.rb +++ b/lib/api/entities/project_import_status.rb @@ -4,6 +4,9 @@ module API module Entities class ProjectImportStatus < ProjectIdentity expose :import_status + expose :correlation_id do |project, _options| + project.import_state.correlation_id + end # TODO: Use `expose_nil` once we upgrade the grape-entity gem expose :import_error, if: lambda { |project, _ops| project.import_state&.last_error } do |project| diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb index f1befe4aff1..1e5d23a8405 100644 --- a/lib/gitlab/sidekiq_cluster/cli.rb +++ b/lib/gitlab/sidekiq_cluster/cli.rb @@ -115,7 +115,7 @@ module Gitlab end def hard_stop_stuck_pids - SidekiqCluster.signal_processes(SidekiqCluster.pids_alive(@processes), :KILL) + SidekiqCluster.signal_processes(SidekiqCluster.pids_alive(@processes), "-KILL") end def wait_for_termination diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 971a77e55be..49f9e66db64 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -10310,6 +10310,9 @@ msgstr "" msgid "GroupSAML|should be a random persistent ID, emails are discouraged" msgstr "" +msgid "GroupSettings|Apply integration settings to all Projects" +msgstr "" + msgid "GroupSettings|Auto DevOps pipeline was updated for the group" msgstr "" @@ -10346,6 +10349,9 @@ msgstr "" msgid "GroupSettings|If the parent group's visibility is lower than the group current visibility, visibility levels for subgroups and projects will be changed to match the new parent group's visibility." msgstr "" +msgid "GroupSettings|Integrations configured here will automatically apply to all projects in this group." +msgstr "" + msgid "GroupSettings|Learn more about badges." msgstr "" diff --git a/spec/controllers/groups/settings/integrations_controller_spec.rb b/spec/controllers/groups/settings/integrations_controller_spec.rb index eee65476dab..6df1ad8a383 100644 --- a/spec/controllers/groups/settings/integrations_controller_spec.rb +++ b/spec/controllers/groups/settings/integrations_controller_spec.rb @@ -11,17 +11,40 @@ describe Groups::Settings::IntegrationsController do sign_in(user) end - describe '#edit' do - context 'when group_level_integrations not enabled' do - it 'returns not_found' do - stub_feature_flags(group_level_integrations: { enabled: false, thing: group }) - - get :edit, params: { group_id: group, id: Service.available_services_names.sample } + describe '#index' do + context 'when user is not owner' do + it 'renders not_found' do + get :index, params: { group_id: group } expect(response).to have_gitlab_http_status(:not_found) end end + context 'when user is owner' do + before do + group.add_owner(user) + end + + context 'when group_level_integrations not enabled' do + it 'returns not_found' do + stub_feature_flags(group_level_integrations: { enabled: false, thing: group }) + + get :index, params: { group_id: group } + + expect(response).to have_gitlab_http_status(:not_found) + end + end + + it 'successfully displays the template' do + get :index, params: { group_id: group } + + expect(response).to have_gitlab_http_status(:ok) + expect(response).to render_template(:index) + end + end + end + + describe '#edit' do context 'when user is not owner' do it 'renders not_found' do get :edit, params: { group_id: group, id: Service.available_services_names.sample } @@ -35,6 +58,16 @@ describe Groups::Settings::IntegrationsController do group.add_owner(user) end + context 'when group_level_integrations not enabled' do + it 'returns not_found' do + stub_feature_flags(group_level_integrations: { enabled: false, thing: group }) + + get :edit, params: { group_id: group, id: Service.available_services_names.sample } + + expect(response).to have_gitlab_http_status(:not_found) + end + end + Service.available_services_names.each do |integration_name| context "#{integration_name}" do it 'successfully displays the template' do diff --git a/spec/lib/gitlab/sidekiq_cluster/cli_spec.rb b/spec/lib/gitlab/sidekiq_cluster/cli_spec.rb index 72727aab601..fa23bf8958c 100644 --- a/spec/lib/gitlab/sidekiq_cluster/cli_spec.rb +++ b/spec/lib/gitlab/sidekiq_cluster/cli_spec.rb @@ -236,7 +236,7 @@ describe Gitlab::SidekiqCluster::CLI do .with([]).and_return([]) expect(Gitlab::SidekiqCluster).to receive(:signal_processes) - .with([], :KILL) + .with([], "-KILL") stub_const("Gitlab::SidekiqCluster::CLI::CHECK_TERMINATE_INTERVAL_SECONDS", 0.1) allow(cli).to receive(:terminate_timeout_seconds) { 1 } @@ -264,7 +264,7 @@ describe Gitlab::SidekiqCluster::CLI do .with(worker_pids).and_return([102]) expect(Gitlab::SidekiqCluster).to receive(:signal_processes) - .with([102], :KILL) + .with([102], "-KILL") cli.run(%w(foo)) diff --git a/spec/lib/gitlab/sidekiq_cluster_spec.rb b/spec/lib/gitlab/sidekiq_cluster_spec.rb index 9316ac29dd6..3f466e9f2a3 100644 --- a/spec/lib/gitlab/sidekiq_cluster_spec.rb +++ b/spec/lib/gitlab/sidekiq_cluster_spec.rb @@ -44,7 +44,7 @@ describe Gitlab::SidekiqCluster do end describe '.signal_processes' do - it 'sends a signal to every thread' do + it 'sends a signal to every given process' do expect(described_class).to receive(:signal).with(1, :INT) described_class.signal_processes([1], :INT) diff --git a/spec/models/project_import_state_spec.rb b/spec/models/project_import_state_spec.rb index 157477767af..720dc4f435f 100644 --- a/spec/models/project_import_state_spec.rb +++ b/spec/models/project_import_state_spec.rb @@ -14,8 +14,8 @@ describe ProjectImportState, type: :model do end describe 'Project import job' do - let(:import_state) { create(:import_state, import_url: generate(:url)) } - let(:project) { import_state.project } + let_it_be(:import_state) { create(:import_state, import_url: generate(:url)) } + let_it_be(:project) { import_state.project } before do allow_any_instance_of(Gitlab::GitalyClient::RepositoryService).to receive(:import_repository) @@ -29,8 +29,16 @@ describe ProjectImportState, type: :model do it 'imports a project', :sidekiq_might_not_need_inline do expect(RepositoryImportWorker).to receive(:perform_async).and_call_original - expect { import_state.schedule }.to change { import_state.jid } - expect(import_state.status).to eq('finished') + expect { import_state.schedule }.to change { import_state.status }.from('none').to('finished') + end + + it 'records job and correlation IDs', :sidekiq_might_not_need_inline do + allow(Labkit::Correlation::CorrelationId).to receive(:current_or_new_id).and_return('abc') + + import_state.schedule + + expect(import_state.jid).to be_an_instance_of(String) + expect(import_state.correlation_id).to eq('abc') end end diff --git a/spec/serializers/diffs_entity_spec.rb b/spec/serializers/diffs_entity_spec.rb index b42240037df..482f9e76c62 100644 --- a/spec/serializers/diffs_entity_spec.rb +++ b/spec/serializers/diffs_entity_spec.rb @@ -45,26 +45,26 @@ describe DiffsEntity do let(:commit) { commits.third } it 'includes commit references for previous and next' do - expect(subject[:commit][:prev_commit_id]).to eq(commits.second.id) - expect(subject[:commit][:next_commit_id]).to eq(commits.fourth.id) + expect(subject[:commit][:next_commit_id]).to eq(commits.second.id) + expect(subject[:commit][:prev_commit_id]).to eq(commits.fourth.id) end end context "when the passed commit is the first in the group" do let(:commit) { commits.first } - it 'includes commit references for nil and next' do - expect(subject[:commit][:prev_commit_id]).to be_nil - expect(subject[:commit][:next_commit_id]).to eq(commits.second.id) + it 'includes commit references for nil and previous commit' do + expect(subject[:commit][:next_commit_id]).to be_nil + expect(subject[:commit][:prev_commit_id]).to eq(commits.second.id) end end context "when the passed commit is the last in the group" do let(:commit) { commits.last } - it 'includes commit references for previous and nil' do - expect(subject[:commit][:prev_commit_id]).to eq(commits[-2].id) - expect(subject[:commit][:next_commit_id]).to be_nil + it 'includes commit references for the next and nil' do + expect(subject[:commit][:next_commit_id]).to eq(commits[-2].id) + expect(subject[:commit][:prev_commit_id]).to be_nil end end end diff --git a/spec/support/shared_contexts/navbar_structure_context.rb b/spec/support/shared_contexts/navbar_structure_context.rb index 2a98855a83f..9ffe13545f7 100644 --- a/spec/support/shared_contexts/navbar_structure_context.rb +++ b/spec/support/shared_contexts/navbar_structure_context.rb @@ -120,6 +120,7 @@ RSpec.shared_context 'group navbar structure' do _('General'), _('Projects'), _('CI / CD'), + _('Integrations'), _('Webhooks'), _('Audit Events') ] |
